Author: Paul

Demonstrating the FHIR Extended Operation Composition/$document

Per the specification, a client can ask a server to generate a fully bundled document from a composition resource. I’ve pulled together a Postman to help demonstrate this feature on the IBM FHIR Server.

Download the Postman

2. Update the SERVER_HOSTNAME

3. Update the Authorization for your username and password

4. Click Tests > Run

5. Click Run

6. You’ll see the tests run.

7. Click on the Tests of Interest, and then check the curl you are interested, such as:

curl --location --request GET 'https://localhost:9443/fhir-server/api/v4/Composition/17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09/$document?persist=true' \
--header 'Content-Type: application/fhir+json' \
--header 'Authorization: Basic .....'

Good Luck with Composition/$document.

	{
	"info": {
	"_postman_id": "6c0a89ea-117d-4d1a-a323-655832225d62",
	"name": "IBM FHIR Server - Composition/$document",
	"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
	},
	"item": [
	{
	"name": "PREP - Create Practitioner",
	"event": [
	{
	"listen": "test",
	"script": {
	"exec": [
	"pm.test(\"Status code (200,201)\", function () {",
	" pm.response.to.have.status(200) \|\| ",
	" pm.response.to.have.status(201);",
	"});"
	],
	"type": "text/javascript"
	}
	}
	],
	"request": {
	"method": "PUT",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"body": {
	"mode": "raw",
	"raw": "{\n \"resourceType\": \"Practitioner\",\n \"id\": \"17b83b98f73-8c735502-4c97-435a-84b9-f8d7a35f1965\",\n \"text\": {\n \"status\": \"generated\",\n \"div\": \"<div xmlns=\\\"http://www.w3.org/1999/xhtml\\\">Assigned, Amanda. SSN:\\n 33344444</div>\"\n },\n \"identifier\": [\n {\n \"type\": {\n \"coding\": [\n {\n \"system\": \"http://terminology.hl7.org/CodeSystem/v2-0203\",\n \"code\": \"SB\"\n }\n ]\n },\n \"system\": \"http://hl7.org/fhir/sid/us-ssn\",\n \"value\": \"33344444\"\n }\n ],\n \"name\": [\n {\n \"use\": \"official\",\n \"family\": \"Assigned\",\n \"given\": [\n \"Amanda\"\n ]\n }\n ],\n \"telecom\": [\n {\n \"system\": \"phone\",\n \"value\": \"555-555-1021\",\n \"use\": \"work\"\n }\n ],\n \"address\": [\n {\n \"use\": \"home\",\n \"line\": [\n \"1020 Healthcare Drive\"\n ]\n }\n ],\n \"gender\": \"female\"\n}"
	},
	"url": {
	"raw": "https://{{SERVER_HOSTNAME}}/fhir-server/api/v4/Practitioner/17b83b98f73-8c735502-4c97-435a-84b9-f8d7a35f1965",
	"protocol": "https",
	"host": [
	"{{SERVER_HOSTNAME}}"
	],
	"path": [
	"fhir-server",
	"api",
	"v4",
	"Practitioner",
	"17b83b98f73-8c735502-4c97-435a-84b9-f8d7a35f1965"
	]
	}
	},
	"response": []
	},
	{
	"name": "PREP - Create Patient",
	"event": [
	{
	"listen": "test",
	"script": {
	"exec": [
	"pm.test(\"Status code (200,201)\", function () {",
	" pm.response.to.have.status(200) \|\| ",
	" pm.response.to.have.status(201);",
	"});"
	],
	"type": "text/javascript"
	}
	}
	],
	"request": {
	"method": "PUT",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"body": {
	"mode": "raw",
	"raw": "{\n \"resourceType\": \"Patient\",\n \"id\": \"17b83b99648-9ef5f08c-d2d5-4a3a-a930-ce45dc2ecaea\",\n \"meta\": {\n \"versionId\": \"1\",\n \"lastUpdated\": \"2021-08-26T18:29:43.113072Z\"\n },\n \"name\": [\n {\n \"family\": \"Doe\",\n \"given\": [\n \"John\"\n ]\n }\n ],\n \"telecom\": [\n {\n \"system\": \"phone\",\n \"value\": \"555-1122\",\n \"use\": \"home\"\n }\n ],\n \"birthDate\": \"1970-01-01\",\n \"generalPractitioner\": [\n {\n \"reference\": \"Practitioner/17b83b98f73-8c735502-4c97-435a-84b9-f8d7a35f1965\"\n }\n ]\n}"
	},
	"url": {
	"raw": "https://{{SERVER_HOSTNAME}}/fhir-server/api/v4/Patient/17b83b99648-9ef5f08c-d2d5-4a3a-a930-ce45dc2ecaea",
	"protocol": "https",
	"host": [
	"{{SERVER_HOSTNAME}}"
	],
	"path": [
	"fhir-server",
	"api",
	"v4",
	"Patient",
	"17b83b99648-9ef5f08c-d2d5-4a3a-a930-ce45dc2ecaea"
	]
	}
	},
	"response": []
	},
	{
	"name": "PREP - Create Observation",
	"event": [
	{
	"listen": "test",
	"script": {
	"exec": [
	"pm.test(\"Status code (200,201)\", function () {",
	" pm.response.to.have.status(200) \|\| ",
	" pm.response.to.have.status(201);",
	"});"
	],
	"type": "text/javascript"
	}
	}
	],
	"request": {
	"method": "PUT",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"body": {
	"mode": "raw",
	"raw": "{\n \"resourceType\": \"Observation\",\n \"id\": \"17b83b99d35-37c44949-7876-4705-8df1-15792f2b05a8\",\n \"meta\": {\n \"versionId\": \"1\",\n \"lastUpdated\": \"2021-08-26T18:29:44.885634Z\"\n },\n \"status\": \"final\",\n \"category\": [\n {\n \"coding\": [\n {\n \"system\": \"http://terminology.hl7.org/CodeSystem/observation-category\",\n \"code\": \"vital-signs\",\n \"display\": \"Vital Signs\"\n }\n ]\n }\n ],\n \"code\": {\n \"coding\": [\n {\n \"system\": \"http://loinc.org\",\n \"code\": \"55284-4\",\n \"display\": \"Blood pressure systolic and diastolic\"\n }\n ],\n \"text\": \"BP\"\n },\n \"subject\": {\n \"reference\": \"Patient/17b83b99648-9ef5f08c-d2d5-4a3a-a930-ce45dc2ecaea\"\n },\n \"component\": [\n {\n \"code\": {\n \"coding\": [\n {\n \"system\": \"http://loinc.org\",\n \"code\": \"8459-0\",\n \"display\": \"Systolic blood pressure--sitting\"\n }\n ]\n },\n \"valueQuantity\": {\n \"value\": 125.0,\n \"unit\": \"mmHg\"\n }\n },\n {\n \"code\": {\n \"coding\": [\n {\n \"system\": \"http://loinc.org\",\n \"code\": \"8453-3\",\n \"display\": \"BP dias--sitting\"\n }\n ]\n },\n \"valueQuantity\": {\n \"value\": 80.0,\n \"unit\": \"mmHg\"\n }\n }\n ]\n}"
	},
	"url": {
	"raw": "https://{{SERVER_HOSTNAME}}/fhir-server/api/v4/Observation/17b83b99d35-37c44949-7876-4705-8df1-15792f2b05a8",
	"protocol": "https",
	"host": [
	"{{SERVER_HOSTNAME}}"
	],
	"path": [
	"fhir-server",
	"api",
	"v4",
	"Observation",
	"17b83b99d35-37c44949-7876-4705-8df1-15792f2b05a8"
	]
	}
	},
	"response": []
	},
	{
	"name": "PREP - Create Composition",
	"event": [
	{
	"listen": "test",
	"script": {
	"exec": [
	"pm.test(\"Status code (200,201)\", function () {",
	" pm.response.to.have.status(200) \|\| ",
	" pm.response.to.have.status(201);",
	"});"
	],
	"type": "text/javascript"
	}
	}
	],
	"request": {
	"method": "PUT",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"body": {
	"mode": "raw",
	"raw": "{\n \"resourceType\": \"Composition\",\n \"id\":\"17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09\",\n \"status\": \"final\",\n \"type\": {\n \"coding\": [\n {\n \"system\": \"http://somesystem.org\",\n \"code\": \"somecode-1234\"\n }\n ]\n },\n \"subject\": {\n \"reference\": \"Patient/17b83b99648-9ef5f08c-d2d5-4a3a-a930-ce45dc2ecaea\"\n },\n \"date\": \"2015-02-14T13:42:00+10:00\",\n \"author\": [\n {\n \"reference\": \"Practitioner/17b83b98f73-8c735502-4c97-435a-84b9-f8d7a35f1965\"\n }\n ],\n \"title\": \"This is the title\",\n \"section\": [\n {\n \"entry\": [\n {\n \"reference\": \"Observation/17b83b99d35-37c44949-7876-4705-8df1-15792f2b05a8\"\n }\n ]\n }\n ]\n}"
	},
	"url": {
	"raw": "https://{{SERVER_HOSTNAME}}/fhir-server/api/v4/Composition/17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09",
	"protocol": "https",
	"host": [
	"{{SERVER_HOSTNAME}}"
	],
	"path": [
	"fhir-server",
	"api",
	"v4",
	"Composition",
	"17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09"
	]
	}
	},
	"response": []
	},
	{
	"name": "1. Execute Composition/$document with Persist",
	"event": [
	{
	"listen": "test",
	"script": {
	"exec": [
	"pm.test(\"Status code (200)\", function () {",
	" pm.response.to.have.status(200);",
	"});",
	"",
	"pm.test(\"Check that there is no headerlocation\", function () {",
	" pm.expect(pm.response.headers.has('Location'));",
	" var headerLocation=postman.getResponseHeader(\"Location\");",
	" pm.collectionVariables.set(\"HEADER_LOCATION\", headerLocation);",
	"});"
	],
	"type": "text/javascript"
	}
	}
	],
	"request": {
	"method": "GET",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"url": {
	"raw": "https://{{SERVER_HOSTNAME}}/fhir-server/api/v4/Composition/17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09/$document?persist=true",
	"protocol": "https",
	"host": [
	"{{SERVER_HOSTNAME}}"
	],
	"path": [
	"fhir-server",
	"api",
	"v4",
	"Composition",
	"17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09",
	"$document"
	],
	"query": [
	{
	"key": "persist",
	"value": "true"
	}
	]
	}
	},
	"response": []
	},
	{
	"name": "2. Get Persisted Composition Bundle",
	"request": {
	"method": "GET",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"url": {
	"raw": "{{HEADER_LOCATION}}",
	"host": [
	"{{HEADER_LOCATION}}"
	]
	}
	},
	"response": []
	},
	{
	"name": "3. Execute Composition/$document Copy",
	"event": [
	{
	"listen": "test",
	"script": {
	"exec": [
	"pm.test(\"Status code (200)\", function () {",
	" pm.response.to.have.status(200);",
	"});",
	"",
	"pm.test(\"Check that there is no headerlocation\", function () {",
	" pm.expect(pm.response.headers.get('Location')).undefined;",
	"});"
	],
	"type": "text/javascript"
	}
	}
	],
	"request": {
	"method": "GET",
	"header": [
	{
	"key": "Content-Type",
	"value": "application/fhir+json",
	"type": "text"
	}
	],
	"url": {
	"raw": "https://{{SERVER_HOSTNAME}}/fhir-server/api/v4/Composition/17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09/$document?persist=false",
	"protocol": "https",
	"host": [
	"{{SERVER_HOSTNAME}}"
	],
	"path": [
	"fhir-server",
	"api",
	"v4",
	"Composition",
	"17b83b99f91-3c0d6274-0498-4fe4-999e-ba8574f85b09",
	"$document"
	],
	"query": [
	{
	"key": "persist",
	"value": "false"
	}
	]
	}
	},
	"response": []
	}
	],
	"auth": {
	"type": "basic",
	"basic": [
	{
	"key": "password",
	"value": "",
	"type": "string"
	},
	{
	"key": "username",
	"value": "fhiruser",
	"type": "string"
	}
	]
	},
	"event": [
	{
	"listen": "prerequest",
	"script": {
	"type": "text/javascript",
	"exec": [
	""
	]
	}
	},
	{
	"listen": "test",
	"script": {
	"type": "text/javascript",
	"exec": [
	""
	]
	}
	}
	],
	"variable": [
	{
	"key": "SERVER_HOSTNAME",
	"value": "localhost:9443"
	},
	{
	"key": "HEADER_LOCATION",
	"value": ""
	}
	]
	}

view raw IBM FHIR Server - Composition-$document.postman_collection.json hosted with

by GitHub

2021-08-27

Recipe: Building a custom IBM FHIR Server container with Bulk Data Parquet Support

Note: Parquet Support is now obsolete.

The IBM FHIR Server has early support for Bulk Data export to the Apache Parquet format using the Apache Spark libraries. New as of version 4.4.0, the export to parquet feature requires:

Apache Spark v3.0 and the IBM Stocator adapter (version 1.1)
the configuration /fhirServer/bulkdata/storageProviders/(source)/enableParquet set to true

The Parquet Bulk Data export is activated using a custom _outputFormat in the export request.

        {
            "name": "_outputFormat",
            "valueString": "application/fhir+parquet"
        },

Let me show you how to build a custom IBM FHIR Server container with parquet support Docker: ibmcom/ibm-fhir-server. It is recommended to use 4.9.0 or higher.

Recipe

Prior to 4.9.0, build the Maven Projects and the Docker Build. You should see [INFO] BUILD SUCCESS after each Maven build, and docker.io/ibmcom/ibm-fhir-server:latest when the Docker build is successful.

mvn clean install -f fhir-examples -B -DskipTests -ntp
mvn clean install -f fhir-parent -B -DskipTests -ntp
docker build -t ibmcom/ibm-fhir-server:latest fhir-install

Download the dependency files for parquet and stocator.

export WORKSPACE=~/git/wffh/2021/fhir
bash ${WORKSPACE}/fhir-bulkdata-webapp/src/main/sh/cache-parquet-deps.sh

Download the fhir-server-config.json

curl -L -o fhir-server-config.json \
    https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server/liberty-config/config/default/fhir-server-config.json
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  8423  100  8423    0     0  40495      0 --:--:-- --:--:-- --:--:-- 40301

Update the fhir-server-config.json to use a IBM COS storage provider with parquet support. You’ll need to update with your HMAC id, internal and external URLs and parquet enabled.

"storageProviders": {
                "default" : {
                    "type": "ibm-cos",
                    "bucketName": "fhir-performance",
                    "location": "us-east",
                    "endpointInternal": "https://s3.us-east.cloud-object-storage.appdomain.cloud",
                    "endpointExternal": "https://s3.us-east.cloud-object-storage.appdomain.cloud",
                    "auth" : {
                        "type": "hmac",
                        "accessKeyId": "key",
                        "secretAccessKey": "secret"
                    },
                    "enableParquet": true,
                    "disableOperationOutcomes": true,
                    "duplicationCheck": false, 
                    "validateResources": false, 
                    "create": false
                }
            }

Start the Docker container, and capture the container id. It’s going to take a few moments to start up as it lays down the test database.

docker run -d -p 9443:9443 -e BOOTSTRAP_DB=true \
  -v $(pwd)/fhir-server-config.json:/config/config/default/fhir-server-config.json \
  -v $(pwd)/deps:/config/userlib/ \
  ibmcom/ibm-fhir-server
3f8e90f20cd42129adc58df8a0295efc3fb2a0f4507350589f71939a072999ae

Check the logs until you see:

docker logs 3f8e90f20cd42129adc58df8a0295efc3fb2a0f4507350589f71939a072999ae
...
[6/16/21, 15:31:34:533 UTC] 0000002a FeatureManage A   CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 17.665 seconds.

Download the Sample Data

curl -L https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server-test/src/test/resources/testdata/everything-operation/Antonia30_Acosta403.json \
-o Antonia30_Acosta403.json

Load the Sample Data bundle to the IBM FHIR Server

curl -k --location --request POST 'https://localhost:9443/fhir-server/api/v4' \
--header 'Content-Type: application/fhir+json' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--data-binary  "@Antonia30_Acosta403.json" -o response.json

Note, DUMMY_PASSWORD should be previously set.

Scan the response.json for any status that is not "status": "201". For example, the status is in the family of User Request Error or Server Side Error.

cat response.json | jq -r '.entry[].response.status' | sort -u
201

Call the export to Parquet operation, and grab the content-location.

curl --location --request GET 'https://localhost:9443/fhir-server/api/v4/$export?_outputFormat=application/fhir%2Bparquet&_type=Patient' \
--header 'X-FHIR-TENANT-ID: default' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--header 'Content-Type: application/json' -k -v

< content-location: https://localhost:9443/fhir-server/api/v4/$bulkdata-status?job=LqzauvqtHSmkpChVHo%2B1MQ

Check the exprot status using the previous URL, and once you see a 200 response, you can go out and use your exported Parquet data.

curl --location --request GET 'https://localhost:9443/fhir-server/api/v4/$bulkdata-status?job=LqzauvqtHSmkpChVHo%2B1MQ' \
--header 'X-FHIR-TENANT-ID: default' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--header 'Content-Type: application/json' -k

{
    "transactionTime": "2021-08-09T00:34:11.594Z",
    "request": "https://localhost:9443/fhir-server/api/v4/$export?_outputFormat=application/fhir%2Bparquet&_type=Patient",
    "requiresAccessToken": false,
    "output": [
        {
            "type": "Patient",
            "url": "https://s3.us-east.cloud-object-storage.appdomain.cloud/fhir-performance/AZ0gsQS05_RqZnHPhj57AfhYSIHU8VzwmnWjDCQdi2I/Patient_1.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=fc85bf9cc1ac49e99e40085f9ba00f77%2F20210809%2Fus-east%2Fs3%2Faws4_request&X-Amz-Date=20210809T003601Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=6d54f677b91d92304caf889eb0a1efbc2b3ebe3d24cefd9c17169b21816d1cdf",
            "count": 1
        }
    ]
}

You can access the files via COS. cos://us-east/fhir-performance/AZ0gsQS05_RqZnHPhj57AfhYSIHU8VzwmnWjDCQdi2I/Patient_1.parquet/part-00000-dba6ec99-7fdb-4674-a202-0452d4435d18-c000-attempt_202108090034065817435166928016302_0003_m_000000_3.snappy.parquet

You now have a working Parquet output.

References

2021-08-08

Recipe: Reindexing with fhir-bucket and the IBM FHIR Server

The IBM FHIR Server enables user defined and implementation guide defined SearchParameter definitions and is constantly improving Specification conformance. Given these dynamic changes, the IBM FHIR Server Search parameter values may require refresh to optimize the Search and Retrieval of the data in the operational data store.

A quick tool to aid in this is fhir-bucket

Let me show you how to run reindexing with the IBM FHIR Server container Docker: ibmcom/ibm-fhir-server. This feature requires 4.9.0 or higher.

Recipe

Prior to 4.9.0, build the Maven Projects and the Docker Build. You should see [INFO] BUILD SUCCESS after each Maven build, and docker.io/ibmcom/ibm-fhir-server:latest when the Docker build is successful.

mvn clean install -f fhir-examples -B -DskipTests -ntp
mvn clean install -f fhir-parent -B -DskipTests -ntp
docker build -t ibmcom/ibm-fhir-server:latest fhir-install

Download the fhir-server-config.json

curl -L -o fhir-server-config.json \
    https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server/liberty-config/config/default/fhir-server-config.json
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  8423  100  8423    0     0  40495      0 --:--:-- --:--:-- --:--:-- 40301

Download the extension-search-parameters.json

curl -L -o extension-search-parameters.json \
    https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server/liberty-config/config/default/extension-search-parameters.json
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   154  100   154    0     0    611      0 --:--:-- --:--:-- --:--:--   611

Start the Docker container, and capture the container id. It’s going to take a few moments to start up as it lays down the test database.

docker run -d -p 9443:9443 -e BOOTSTRAP_DB=true \
  -v $(pwd)/fhir-server-config.json:/config/config/default/fhir-server-config.json \
  -v $(pwd)/extension-search-parameters.json:/config/config/default/extension-search-parameters.json \
  ibmcom/ibm-fhir-server
9c44ad49784a6ca6456799ab63aea460fc564cc4a9bd5f4fef1aeae4b82225bd

Check the logs until you see:

docker logs 9c44ad49784a6ca6456799ab63aea460fc564cc4a9bd5f4fef1aeae4b82225bd
...
[6/16/21, 15:31:34:533 UTC] 0000002a FeatureManage A   CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 17.665 seconds.

Download the Sample Data

curl -L https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server-test/src/test/resources/testdata/everything-operation/Antonia30_Acosta403.json \
-o Antonia30_Acosta403.json

Load the Sample Data bundle to the IBM FHIR Server

curl -k --location --request POST 'https://localhost:9443/fhir-server/api/v4' \
--header 'Content-Type: application/fhir+json' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--data-binary  "@Antonia30_Acosta403.json" -o response.json

Note, DUMMY_PASSWORD should be previously set.

Scan the response.json for any status that is not "status": "201". For example, the status is in the family of User Request Error or Server Side Error.

cat response.json | jq -r '.entry[].response.status' | sort -u
201

Search maiden-name = Graciela518 Badillo851, and you’ll see the reindexing completes and the search succeeds.

curl --location --request GET 'https://localhost:9443/fhir-server/api/v4/Patient?maiden-name=Graciela518 Badillo851' \
--header 'X-FHIR-TENANT-ID: default' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--header 'Content-Type: application/json' -k

{ 
    "resourceType":"OperationOutcome",
    "id":"ac-11-0-3-14c966ad-8c8b-438d-86ad-de4b2fdb27b0",
    "issue":[
        {
            "severity":"fatal",
            "code":"invalid",
            "details": {
                "text":"Search parameter 'maiden-name' for resource type 'Patient' was not found."
            },
            "expression":["<empty>"]
        }
    ]
}

Download the extension-search-parameters.json with an example expression.

curl -L -o extension-search-parameters.json \
    https://gist.githubusercontent.com/prb112/03f0d77f72685180868d2f31a1070ebe/raw/14fda1cb86dda74095faa5e32fec6f9226ffb587/extension-search-parameters.json
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   752  100   752    0     0   2231      0 --:--:-- --:--:-- --:--:--  2231

Post 4.9.0’s release, download the fhir-bucket-cli

curl -L -o fhir-bucket-4.9.0-cli.jar \
    https://repo1.maven.org/maven2/com/ibm/fhir/fhir-bucket/4.9.0/fhir-bucket-4.9.0-cli.jar
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 46.7M  100 46.7M    0     0  12.2M      0  0:00:03  0:00:03 --:--:-- 12.2M

Pre 4.9.0’s release, copy the fhir-bucket-cli from the .m2 repository.

cp ~/.m2/repository/com/ibm/fhir/fhir-bucket/4.9.0-SNAPSHOT/fhir-bucket-4.9.0-SNAPSHOT-cli.jar fhir-bucket-4.9.0-cli.jar

Download the client truststore

curl -L -o fhirClientTrustStore.p12 \
    https://github.com/IBM/FHIR/raw/main/fhir-server-test/src/test/resources/fhirClientTrustStore.p12
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   174  100   174    0     0    661      0 --:--:-- --:--:-- --:--:--   661
100  5946  100  5946    0     0  12651      0 --:--:-- --:--:-- --:--:-- 12651

Download the fhir-bucket logging.properties

curl -L -o logging.properties \
    https://raw.githubusercontent.com/IBM/FHIR/main/fhir-bucket/logging.properties
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   688  100   688    0     0   3245      0 --:--:-- --:--:-- --:--:--  3245

Create the bucket.properties file

cat << EOF > bucket.properties
fhir.server.host=localhost
fhir.server.port=9443
fhir.server.user=fhiruser
fhir.server.pass=change-password
fhir.server.endpoint=/fhir-server/api/v4/
truststore=fhirClientTrustStore.p12
truststore.pass=change-password
read.timeout=125000
connect.timeout=20000
pool.connections.max=400
disable.hostname.verification=true
EOF

Execute the fhir-bucket reindex operation with the current date-time (e.g. now is 2021-12-01T00:00:00Z)

export JAR="fhir-bucket-4.9.0-cli.jar"
java -Djava.util.logging.config.file=logging.properties -jar "${JAR}" \
    --fhir-properties bucket.properties \
    --tenant-name default \
    --max-concurrent-fhir-requests 100 \
    --no-scan \
    --reindex-tstamp 2021-12-01T00:00:00Z \
    --reindex-resource-count 50 \
    --reindex-concurrent-requests 20 \
    --reindex-client-side-driven

Wait until the reindexing threads complete.

2021-08-05 15:28:05.629 00000001    INFO x.ClientDrivenReindexOperation Starting monitor thread
2021-08-05 15:28:07.864 0000001b    INFO x.ClientDrivenReindexOperation called $retrieve-index: 200 OK [took 0.851 s]
2021-08-05 15:28:07.872 0000001b    INFO x.ClientDrivenReindexOperation Index IDs available for processing - filling worker pool
2021-08-05 15:28:09.145 0000001d    INFO x.ClientDrivenReindexOperation called $reindex (indexIds=2,...): 200 OK [took 1.268 s]
...
2021-08-05 15:28:27.946 0000001b    INFO x.ClientDrivenReindexOperation called $retrieve-index (afterIndexId=1802): 200 OK [took 0.053 s]
2021-08-05 15:28:27.949 0000001b    INFO x.ClientDrivenReindexOperation No more index IDs to retrieve
2021-08-05 15:28:27.950 0000001b    INFO x.ClientDrivenReindexOperation Nothing left to do, so tell all the worker threads to exit
2021-08-05 15:28:27.951 0000001b    INFO x.ClientDrivenReindexOperation Waiting for 20 threads to complete before exiting
2021-08-05 15:28:32.952 0000001b    INFO x.ClientDrivenReindexOperation Reindexing was completed
2021-08-05 15:29:32.891 0000001a    INFO   com.ibm.fhir.bucket.app.Main Stopping all services

Search maiden-name = Graciela518 Badillo851, and you’ll see the reindexing completes and the search succeeds.

curl --location --request GET 'https://localhost:9443/fhir-server/api/v4/Patient?maiden-name=Graciela518 Badillo851' \
--header 'X-FHIR-TENANT-ID: default' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--header 'Content-Type: application/json' -k

    {
            "id": "17b17befcde-8fb6aca7-9008-4682-ae7f-4c3c5a91c7e2",
            "fullUrl": "https://localhost:9443/fhir-server/api/v4/Patient/17b17befcde-8fb6aca7-9008-4682-ae7f-4c3c5a91c7e2",
            "resource": {
                "resourceType": "Patient",
                "id": "17b17befcde-8fb6aca7-9008-4682-ae7f-4c3c5a91c7e2",
                "meta": {
                    "versionId": "1",
                    "lastUpdated": "2021-08-05T19:16:37.89885Z"
                },
                "text": {
                    "status": "generated",
                    "div": "<div xmlns=\"http://www.w3.org/1999/xhtml\">Generated by <a href=\"https://github.com/synthetichealth/synthea\">Synthea</a>.Version identifier: master-branch-latest-2-g8e7e92c\n .   Person seed: 7105486291024734541  Population seed: 0</div>"
                },
                "extension": [
                    {
                        "url": "http://hl7.org/fhir/StructureDefinition/patient-mothersMaidenName",
                        "valueString": "Graciela518 Badillo851"
                    },
                    {
                        "url": "http://hl7.org/fhir/StructureDefinition/patient-birthPlace",
                        "valueAddress": {
                            "city": "Caguas",
                            "state": "Puerto Rico",
                            "country": "PR"
                        }
                    },
                    {
                        "url": "http://synthetichealth.github.io/synthea/disability-adjusted-life-years",
                        "valueDecimal": 1.6165934938317164
                    },
                    {
                        "url": "http://synthetichealth.github.io/synthea/quality-adjusted-life-years",
                        "valueDecimal": 47.383406506168288
                    },
                    {
                        "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex",
                        "valueCode": "F"
                    },
                    {
                        "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-birthsex",
                        "valueCode": "M"
                    }
                ],
                "identifier": [
                    {
                        "system": "https://github.com/synthetichealth/synthea",
                        "value": "bd958c64-56b6-f206-b03d-8f4b8a417215"
                    },
                    {
                        "type": {
                            "coding": [
                                {
                                    "system": "http://terminology.hl7.org/CodeSystem/v2-0203",
                                    "code": "MR",
                                    "display": "Medical Record Number"
                                }
                            ],
                            "text": "Medical Record Number"
                        },
                        "system": "http://hospital.smarthealthit.org",
                        "value": "bd958c64-56b6-f206-b03d-8f4b8a417215"
                    },
                    {
                        "type": {
                            "coding": [
                                {
                                    "system": "http://terminology.hl7.org/CodeSystem/v2-0203",
                                    "code": "SS",
                                    "display": "Social Security Number"
                                }
                            ],
                            "text": "Social Security Number"
                        },
                        "system": "http://hl7.org/fhir/sid/us-ssn",
                        "value": "999-93-6090"
                    },
            }
        ]

References

IBM FHIR Server: User’s Guide

2021-08-05

Recipe: Running the IBM FHIR Server behind a Reverse Proxy

A common deployment pattern for the IBM FHIR Server is to run the Application Server behind a reverse proxy (e.g. OpenShift Route, Kubernetes Ingress, NGINX or API Gateway). By default, the IBM FHIR Server runs under the https://localhost:9443/fhir-server/api/v4 context root and URI. With a modest configuration change, one can change to a context root (baseUrl) or use the X-FHIR-FORWARDED-URL to forward the incoming url to the IBM FHIR Server (expected to be from a trusted reverse proxy).

In #1965, the fhirServer/core/externalBaseUrl is a tenant aware configuration that sets the context root and base URL. Note, the fhirServer/core/externalBaseUrl overrides the incomingUrl from X-FHIR-FORWARDED-URL.

This document outlines how to set the externalBaseUrl for the IBM FHIR Server.

Let me show you how to add a set the externalBaseUrl to an IBM FHIR Server container Docker: ibmcom/ibm-fhir-server. This feature requires 4.9.0 or higher.

Recipe

Prior to 4.9.0, build the Maven Projects and the Docker Build. You should see [INFO] BUILD SUCCESS after each Maven build, and docker.io/ibmcom/ibm-fhir-server:latest when the Docker build is successful.

mvn clean install -f fhir-examples -B -DskipTests -ntp
mvn clean install -f fhir-parent -B -DskipTests -ntp
docker build -t ibmcom/ibm-fhir-server:latest fhir-install</code></pre>

Download the fhir-server-config.json

curl -L -o fhir-server-config.json \
    https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server/liberty-config/config/default/fhir-server-config.json
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  8423  100  8423    0     0  35095      0 --:--:-- --:--:-- --:--:-- 34950

Update the fhir-server-config.json at path fhirServer/core/externalBaseUrl to https://chocolate.fudge.

"externalBaseUrl": "https://chocolate.fudge"

Start the Docker container, and capture the container id. It’s going to take a few moments to start up as it lays down the test database.

docker run -d -p 9443:9443 -e BOOTSTRAP_DB=true \
  -v $(pwd)/fhir-server-config.json:/config/config/default/fhir-server-config.json \
  ibmcom/ibm-fhir-server
a096978867195ff6e610c36cdba77ff423c31c0ad488a7390f42cef6e89e7fd0

Check the logs until you see:

docker logs a096978867195ff6e610c36cdba77ff423c31c0ad488a7390f42cef6e89e7fd0
...
[6/16/21, 15:31:34:533 UTC] 0000002a FeatureManage A   CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 17.665 seconds.

Download the Sample Data

curl -L https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server-test/src/test/resources/testdata/everything-operation/Antonia30_Acosta403.json -o Antonia30_Acosta403.json

Load the Sample Data bundle to the IBM FHIR Server

curl -k --location --request POST 'https://localhost:9443/fhir-server/api/v4' \
--header 'Content-Type: application/fhir+json' \
--user "fhiruser:${DUMMY_PASSWORD}" \
--data-binary  "@Antonia30_Acosta403.json" -o response.json

Scan the response.json for any status that is not "status": "201". For example, the status is in the family of User Request Error or Server Side Error.
Check the patient and you’ll see the self and next relation links include https://chocolate.fudge

curl -k --location --request GET 'https://localhost:9443/fhir-server/api/v4/Patient' \
--header 'Content-Type: application/fhir+json' \
--user "fhiruser:${DUMMY_PASSWORD}" \

{
    "resourceType": "Bundle",
    "id": "37c5abc7-d3e7-4506-b596-9725c59f9b6b",
    "type": "searchset",
    "total": 23,
    "link": [
        {
            "relation": "self",
            "url": "https://chocolate.fudge/Patient?_count=10&_page=1"
        },
        {
            "relation": "next",
            "url": "https://chocolate.fudge/Patient?_count=10&_page=2"
        }
    ],
    "entry": [
        {
            "id": "17b123f9a79-bd2011c1-6606-4617-90ed-3187790955b8",
            "fullUrl": "https://chocolate.fudge/Patient/17b123f9a79-bd2011c1-6606-4617-90ed-3187790955b8",
            "resource": {
                "resourceType": "Patient",
                "id": "17b123f9a79-bd2011c1-6606-4617-90ed-3187790955b8",
                "meta": {
                    "versionId": "1",
                    "lastUpdated": "2021-08-04T17:39:23.385314Z",
                    "tag": [
                        {
                            "system": "http://terminology.hl7.org/CodeSystem/v3-ActReason",
                            "code": "HTEST",
                            "display": "test health data"
                        }
                    ]
                }
            }
        }
    ]
}

References

IBM FHIR Server: User’s Guide

2021-08-05

Postgres and Vacuum with the IBM FHIR Server: Debugging Details

The IBM FHIR Server stores resources in the PostgreSQL database and uses a relational model to store historical FHIR Resource and enable search on the latest FHIR resources. The resource data is spread in a relational model that is occasionally tweaked in order to improve search or optimize the retrieval using the relational model.

In the IBM FHIR Server Performance Guide, the guide outlines some important alterations to the tables that facilitate an optimized Postgres instance. The guide suggests altering, per your providers recommendation, autovacuum_vacuum_cost_limit, autovacuum_vacuum_scale_factor and autovacuum_vacuum_threshold in order to optimize the VACUUM process. With the IBM FHIR Server fhir-persistence-schema-cli, autovacuum_vacuum_scale_factor is not automatically configured, and not recommended on Databases for Postgres on IBM Cloud.

As Postgres uses "multi-version concurrency control (MVCC) to ensure that data remains consistent and accessible in high-concurrency environments", each transaction runs on a snapshot, and needs to be reconciled so dead_rows are removed – vacuumed. The VACUUM process manages dead rows and disk usage (reuse). The VACUUM process (autovacuum) frequently runs – gathering statistics and reconciling the maintenance of the table statitstics and data.

To check for tables that need vacuuming:

 SELECT relname AS "table_name",
        n_tup_ins AS "inserts",
        n_tup_upd AS "updates",
        n_tup_del AS "deletes",
        n_live_tup AS "live_tuples",
        n_dead_tup AS "dead_tuples"
   FROM pg_stat_user_tables
  WHERE schemaname = 'fhirdata'
    AND (relname = 'logical_resources' OR relname LIKE '%_values')
    AND n_dead_tup > 0;

Then a database administrator runs – VACUUM FULL FHIRDATA.PROCEDURE_RESOURCE_TOKEN_REFS; to execute a vacuum, which runs in the background.

While the VACUUM process is running, the pg_stat_progress_vacuum view can be queried to see worker process.

SELECT * 
FROM pg_stat_progress_vacuum;

If you need to update a specific tables settings, you can run with --vacuum-table-name.

java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \
--db-type postgresql --prop db.host=localhost --prop db.port=5432 \
--prop db.database=fhirdb --schema-name fhirdata \
--prop user=fhiradmin --prop password=passw0rd \
--update-vacuum --vacuum-cost-limit 2000 --vacuum-threshold 1000 \
--vacuum-scale-factor 0.01 --vacuum-table-name LOGICAL_RESOURCES

To update all tables in a schema, you can run without the table parameter. If you omit any value, the defaults are picked as described in the Performance guide.

If you hit a lock (ShareUpdateExclusiveLock), the VACUUM worker process is currently churning on the table, and the ALTER statement is waiting.

wait_type = Lock relation Waiting to acquire a lock on a relation.
wait_lock_type – ShareUpdateExclusiveLock Acquired by VACUUM and conflicts with ALTER

CHeck for the Blocking PID, and grab the blocking_pid.

 -- list bad connections
   SELECT blocked_locks.pid     AS blocked_pid,
         blocked_activity.usename  AS blocked_user,
         blocking_locks.pid     AS blocking_pid,
         blocking_activity.usename AS blocking_user,
         blocked_activity.query    AS blocked_statement,
         blocking_activity.query   AS current_statement_in_blocking_process,
         blocked_activity.application_name AS blocked_application,
         blocking_activity.application_name AS blocking_application
   FROM  pg_catalog.pg_locks         blocked_locks
    JOIN pg_catalog.pg_stat_activity blocked_activity  ON blocked_activity.pid = blocked_locks.pid
    JOIN pg_catalog.pg_locks         blocking_locks 
        ON blocking_locks.locktype = blocked_locks.locktype
        AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
        AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
        AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
        AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
        AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
        AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
        AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
        AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
        AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
        AND blocking_locks.pid != blocked_locks.pid
    JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
   WHERE NOT blocked_locks.GRANTED and blocked_activity.usename = 'fhirserver'

Try canceling the PID, SELECT pg_cancel_backend(205384);

Else, cancel the current Transaction the blocked pid:

-- cancel the blocking trannsaction/pid (hard stop)
SELECT pg_cancel_backend(blocked_locks.pid)     AS blocked_pid,
         blocked_activity.usename  AS blocked_user,
         blocking_locks.pid     AS blocking_pid,
         blocking_activity.usename AS blocking_user,
         blocked_activity.query    AS blocked_statement,
         blocking_activity.query   AS current_statement_in_blocking_process,
         blocked_activity.application_name AS blocked_application,
         blocking_activity.application_name AS blocking_application
   FROM  pg_catalog.pg_locks         blocked_locks
    JOIN pg_catalog.pg_stat_activity blocked_activity  ON blocked_activity.pid = blocked_locks.pid
    JOIN pg_catalog.pg_locks         blocking_locks 
        ON blocking_locks.locktype = blocked_locks.locktype
        AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
        AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
        AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
        AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
        AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
        AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
        AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
        AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
        AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
        AND blocking_locks.pid != blocked_locks.pid
    JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
   WHERE NOT blocked_locks.GRANTED and blocked_activity.usename = 'fhirserver'

Now, wait until the VACUUM finishes, and then execute a new ALTER.

You should be all set at this point.

Check wait_type

-- Check Wait Type
SELECT 
    waiting.locktype           AS waiting_locktype,
    waiting.relation::regclass AS waiting_table,
    waiting_stm.query          AS waiting_query,
    waiting.mode               AS waiting_mode,
    waiting.pid                AS waiting_pid,
    other.locktype             AS other_locktype,
    other.relation::regclass   AS other_table,
    other_stm.query            AS other_query,
    other.mode                 AS other_mode,
    other.pid                  AS other_pid,
    other.granted              AS other_granted
FROM
    pg_catalog.pg_locks AS waiting
JOIN
    pg_catalog.pg_stat_activity AS waiting_stm
    ON (
        waiting_stm.pid = waiting.pid
    )
JOIN
    pg_catalog.pg_locks AS other
    ON (
        (
            waiting."database" = other."database"
        AND waiting.relation  = other.relation
        )
        OR waiting.transactionid = other.transactionid
    )
JOIN
    pg_catalog.pg_stat_activity AS other_stm
    ON (
        other_stm.pid = other.pid
    )
WHERE
    NOT waiting.granted
AND
    waiting.pid <> other.pid   

SELECT
  schemaname, relname,
  last_vacuum, last_autovacuum,
  vacuum_count, autovacuum_count, *
FROM pg_stat_user_tables
WHERE schemaname = 'fhirdata' AND relname = 'observation_date_values';

Check with locks

SELECT now()::time, query, backend_start, xact_start, query_start,
         state_change, state,
         now()::time - state_change::time AS locked_since,
         pid, wait_event_type, wait_event
  FROM pg_stat_activity
  WHERE wait_event_type IS NOT NULL and wait_event_type = 'Lock'
ORDER BY locked_since DESC;

Check a PID

SELECT a.usename, a.application_name, a.datname, a.query,
         l.granted, l.mode, transactionid
    FROM pg_locks l
    JOIN pg_stat_activity a ON a.pid = l.pid
    WHERE granted = false AND a.pid = 327589;

Check a tansaction

SELECT a.usename, a.application_name, a.datname, a.query,
        l.granted, l.mode, transactionid,
        now()::time - a.state_change::time AS acquired_since,
        a.pid
   FROM pg_locks l
   JOIN pg_stat_activity a ON a.pid = l.pid
   WHERE granted = true AND transactionid = 3031;

Reference

2021-07-27

Recipe: Testing Restricted Policies with OpenShift and the IBM FHIR Server Schema Tool
Docker: ibmcom/ibm-fhir-schematool supports onboarding and offboarding of a schema in support of the IBM FHIR Server. I am working on a project that runs on OpenShift using CodeReadyContainers, and I needed to test the image with a restricted policy.

Note, these are roughly my notes from testing, and converted to a post.

Recipe
1. Start up Code Ready Containers or create your OpenShift environment.
2. Connect to the Docker registry in CRC
```
docker login -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
```
1. Build your Docker image with a new tag
```
docker build . -t default/ibm-fhir-schematool:latest
docker push default/ibm-fhir-schematool:latest
```
1. Login to OpenShift
```
oc login -u $(oc whoami) -p $(oc whoami -t)
```
1. Create a new persistence.json using one of the examples as a template
```
oc create secret generic persistence-json --from-file=persistence-onboard-example.json -n=default
```
1. Add the restricted policy to the developer user.
```
oc adm policy add-scc-to-user restricted developer
```
1. Add the developer user to the default namespace.
```
oc adm policy add-role-to-user edit developer -n=default
```
1. Make the locak registry lookup use relative names
```
oc set image-lookup  --all
```
1. Create a new pod.yaml
```
apiVersion: v1
kind: Pod
metadata:
  name: ibm-fhir-server-schematool
spec:
  containers:
    - name: test-container
      image: default/ibm-fhir-schematool
      env:
        - name: ENV_TOOL_INPUT
          valueFrom:
            secretKeyRef:
              name: persistence-json
              key: persistence-onboard-example.json
```
1. Create the pod.
```
oc apply -f pod.yaml --as=developer  -n=default
```
1. Check the logs, and you should see a successful run.
```
oc logs pod/ibm-fhir-server-schematool
```
References
2021-07-27
Syncing Git Tags across Repositories
To sync the tags from one repository to another, you can do the following:
1. Clone the repository that you want the tags to be in, and change to that cloned repository.
```
git clone git@github.com:prb112/FHIR.git
cd FHIR
```
1. Add a remote to the original repository.
```
git remote add fhiro git@github.com:IBM/FHIR.git
```
1. Check that the remote is there (fhiro).
```
git remote -v
fhiro	git@github.com:IBM/FHIR.git (fetch)
fhiro	git@github.com:IBM/FHIR.git (push)
origin	git@github.com:prb112/FHIR.git (fetch)
origin	git@github.com:prb112/FHIR.git (push)
```
1. Fetch the original tags.
```
git fetch fhiro --prune --tags
```
1. Check the tags are shown.
```
git tag --list
```
1. Remove the remote.
```
git remote remove fhiro
```
1. Check that the remote is removed.
```
git remote -v
origin	git@github.com:prb112/FHIR.git (fetch)
origin	git@github.com:prb112/FHIR.git (push)
```
1. Push the tags to the new destination.
```
git push --tags
```
Note I did disable the GitHub Actions prior, and re-enabled after.

References
- Working with Remotes
- In git, how do I sync my tags against a remote server?
2021-07-13

Recipe: IBM FHIR Server – Adding a Custom Extended Operation to the IBM FHIR Server

The IBM FHIR Server has support for extended operations beyond the standard C-R-U-D. The Extended Operations are supported at the System, Resource, and Instance levels. Operations are packaged as JAR files, and the IBM FHIR Server loads the specific Operation using the Java ServiceLoader framework at startup. More implementation specific details are at FHIROperationFramework.

The IBM FHIR Server repository builds a number of operations, release to Maven Repository, and bundles a subset automatically with the Docker container. Some of the bundles operations include Healthcheck, Terminology, BulkData Import, BulkData Export, Reindex, Everything, Erase, Convert, Document and Validate. The IBM FHIR Server maintains the Capability Statement which advertises the System, Type and Instance operations on the metadata endpoint.

The IBM FHIR Server can take any configured custom extended operation, such as HelloOperation which is part of the fhir-operation-test.

This document outlines how to add a custom extended operation to the IBM FHIR Server.

Let me show you how to add a custom extended operation to an IBM FHIR Server container Docker: ibmcom/ibm-fhir-server.

Recipe

Download the fhir-server-config.json

curl -L -o fhir-server-config.json \
    https://raw.githubusercontent.com/IBM/FHIR/main/fhir-server/liberty-config/config/default/fhir-server-config.json
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  8423  100  8423    0     0  35095      0 --:--:-- --:--:-- --:--:-- 34950

Download the fhir-operation-test-*-tests.jar. Note, you should pick the most recent release.

curl -L -o fhir-operation-test-4.8.3.jar \
    https://repo1.maven.org/maven2/com/ibm/fhir/fhir-operation-test/4.8.3/fhir-operation-test-4.8.3-tests.jar
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 12393  100 12393    0     0  58734      0 --:--:-- --:--:-- --:--:-- 58734

Create a directory called userlib, don’t put the jar in the userlib yet.

mkdir -p userlib

Start the Docker container, and capture the container id. It’s going to take a few moments to start up as it lays down the test database.

docker run -d -p 9443:9443 -e BOOTSTRAP_DB=true \
  -v $(pwd)/fhir-server-config.json:/config/config/default/fhir-server-config.json \
  -v $(pwd)/userlib/:/config/userlib/ ibmcom/ibm-fhir-server
a096978867195ff6e610c36cdba77ff423c31c0ad488a7390f42cef6e89e7fd0

Check the logs until you see:

docker logs a096978867195ff6e610c36cdba77ff423c31c0ad488a7390f42cef6e89e7fd0
...
[6/16/21, 15:31:34:533 UTC] 0000002a FeatureManage A   CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 17.665 seconds.

Check the Type and Instance operations, you can check using this curl/jq command.

curl -k --location --request GET 'https://localhost:9443/fhir-server/api/v4/metadata' \
    | jq -r '.rest[].resource[].operation[].name' | sort -u
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 1096k    0 1096k    0     0   103k      0 --:--:--  0:00:10 --:--:--  259k
apply
closure
convert
document
erase
everything
expand
export
lookup
reindex
subsumes
translate
validate
validate-code

Note, that the hello operation is not displayed. The CapabilityStatement is best looked at selectively, since it’s such a big document, thus the use of jq.

To check the System Level operations, you can check using this curl/jq command.

curl -k --location --request GET 'https://localhost:9443/fhir-server/api/v4/metadata' \
    | jq -r '.rest[].operation[].name'
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 1096k    0 1096k    0     0  2592k      0 --:--:-- --:--:-- --:--:-- 2592k
bulkdata-status
closure
convert
export
healthcheck
import
reindex

Note, that the hello operation is not displayed.

Copy over the fhir-operation-test-*-tests.jar to userlib/.
Restart the docker container

docker restart a096978867195ff6e610c36cdba77ff423c31c0ad488a7390f42cef6e89e7fd0

Check the logs until you see:

docker logs a096978867195ff6e610c36cdba77ff423c31c0ad488a7390f42cef6e89e7fd0
...
[6/16/21, 15:31:34:533 UTC] 0000002a FeatureManage A   CWWKF0011I: The defaultServer server is ready to run a smarter planet. The defaultServer server started in 17.665 seconds.

To check the System Level operations, you can check using this curl/jq command.

curl -k --location --request GET 'https://localhost:9443/fhir-server/api/v4/metadata' \
    | jq -r '.rest[].operation[].name'
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 1096k    0 1096k    0     0  2562k      0 --:--:-- --:--:-- --:--:-- 2556k
bulkdata-status
closure
convert
export
healthcheck
hello
import
reindex

You should see the hello operation listed.

Now you can execute the hello operation.

curl -k --location --request GET 'https://localhost:9443/fhir-server/api/v4/$hello' \
    -v -u fhiruser:change-password
...
* Connection state changed (MAX_CONCURRENT_STREAMS == 200)!
< HTTP/2 200 
< date: Fri, 09 Jul 2021 23:48:33 GMT
< content-length: 0
< content-language: en-US
< 
* Connection #0 to host localhost left intact
* Closing connection 0

You’ve now seen how to add a jar file to the userlib in a running container that adds the custom operation HelloOperation.

References

2021-07-09

Determining Function Signatures with Postgres

I had duplicate Postgres base signatures, and I needed to diagnose the reasons why it was failing to update. The following are helpful for diagnosing functions:

	SELECT routines.routine_name, parameters.data_type, parameters.ordinal_position, parameters.parameter_mode
	FROM information_schema.routines
	LEFT JOIN information_schema.parameters ON routines.specific_name=parameters.specific_name
	WHERE routines.specific_schema='fhirdata'
	AND routines.routine_name = 'add_any_resource'
	ORDER BY routines.routine_name, parameters.ordinal_position;

view raw list-functions-and-parameters-1.sql hosted with

by GitHub

	"add_any_resource" "character varying" 1 "IN"
	"add_any_resource" "character varying" 2 "IN"
	"add_any_resource" "bytea" 3 "IN"
	"add_any_resource" "timestamp without time zone" 4 "IN"
	"add_any_resource" "character" 5 "IN"
	"add_any_resource" "character varying" 6 "IN"
	"add_any_resource" "integer" 7 "IN"
	"add_any_resource" "character varying" 8 "IN"
	"add_any_resource" "bigint" 9 "OUT"
	"add_any_resource" "character varying" 10 "OUT"

view raw list-functions-and-parameters-output.sql hosted with

by GitHub

	select p.oid::regproc, pg_get_function_identity_arguments(p.oid)
	FROM pg_catalog.pg_proc p
	WHERE p.oid::regproc::text = 'fhirdata.erase_resource';

view raw list-functions1.sql hosted with

by GitHub

fhirdata.erase_resource | p_resource_type character varying, p_logical_id character varying, OUT o_deleted bigint

view raw list-functions2.sql hosted with

by GitHub

References

StackOverflow DROP FUNCTION without knowing the number/type of parameters?

StackOverflow: pg_proc

2021-06-30

GPG complains about No keyserver available

When I wanted to deploy my key to openpgp, I hit the issue where it said No keyserver available:

$ gpg  --verbose --keyserver hkps://keys.openpgp.org  --send-keys KEYDYID
gpg: Note: RFC4880bis features are enabled.
gpg: sending key KEYDYID to hkps://keys.openpgp.org
gpg: keyserver send failed: No keyserver available
gpg: keyserver send failed: No keyserver available

If you hit this, you can ps -ef dirmgr and then kill -9 the pid for the dirmngr.

Restart the dirmngr --debug-all --daemon --standard-resolver

Check the output for any errors (in my case a TLS issue – TLS connection authentication failed: General error)

09:02:27-homedir@machine:~$ dirmngr --debug-all --daemon --standard-resolver
dirmngr[58503]: reading options from '/Users/homedir/.gnupg/dirmngr.conf'
dirmngr[58503]: reading options from '[cmdline]'
dirmngr[58503]: enabled debug flags: x509 crypto memory cache memstat hashing ipc dns network lookup extprog
dirmngr[58503]: listening on socket '/Users/homedir/.gnupg/S.dirmngr'
DIRMNGR_INFO=/Users/homedir/.gnupg/S.dirmngr:58504:1; export DIRMNGR_INFO;
09:02:52-homedir@machine:~$ dirmngr[58504.0]: permanently loaded certificates: 133
dirmngr[58504.0]:     runtime cached certificates: 0
dirmngr[58504.0]:            trusted certificates: 133 (132,0,0,1)
dirmngr[58504.4]: handler for fd 4 started
dirmngr[58504.4]: DBG: chan_4 -> # Home: /Users/homedir/.gnupg
dirmngr[58504.4]: DBG: chan_4 -> # Config: /Users/homedir/.gnupg/dirmngr.conf
dirmngr[58504.4]: DBG: chan_4 -> OK Dirmngr 2.3.1 at your service
dirmngr[58504.4]: DBG: END Certificate
dirmngr[58504.4]: DBG: BEGIN Certificate 'server[2]':
dirmngr[58504.4]: DBG:      serial: 4001772137D4E942B8EE76AA3C640AB7
dirmngr[58504.4]: DBG:   notBefore: 2021-01-20 19:14:03
dirmngr[58504.4]: DBG:    notAfter: 2024-09-30 18:14:03
dirmngr[58504.4]: DBG:      issuer: CN=DST Root CA X3,O=Digital Signature Trust Co.
dirmngr[58504.4]: DBG:     subject: CN=ISRG Root X1,O=Internet Security Research Group,C=US
dirmngr[58504.4]: DBG:   hash algo: 1.2.840.113549.1.1.11
dirmngr[58504.4]: DBG:   SHA1 fingerprint: 933C6DDEE95C9C41A40F9F50493D82BE03AD87BF
dirmngr[58504.4]: DBG: END Certificate
dirmngr[58504.4]: TLS connection authentication failed: General error
dirmngr[58504.4]: error connecting to 'http://keys.openpgp.org:80': General error
dirmngr[58504.4]: command 'KS_PUT' failed: General error <Unspecified source>
dirmngr[58504.4]: DBG: chan_4 -> ERR 1 General error <Unspecified source>
dirmngr[58504.4]: DBG: chan_4 <- BYE
dirmngr[58504.4]: DBG: chan_4 -> OK closing connection
dirmngr[58504.4]: handler for fd 4 terminated

Create the file ~/.gnupg/dirmngr.conf with the following contents

keyserver hkps://keys.openpgp.org
hkp-cacert /Users/homedir/.gnupg/my.pem

Download the openpgp ceritifcate

echo "" | openssl s_client -showcerts -prexit -connect keys.openpgp.org:443 2> /dev/null \
    | sed -n -e '/BEGIN CERTIFICATE/,/END CERTIFICATE/ p' > ~/.gnupg/my.pem

Restart the dirmngr

Re-execute the gpg command

09:02:58-paulbastide@pauls-mbp:~$ gpg --keyserver hkp://keys.openpgp.org:80 --send-keys KEYDYID
gpg: sending key KEYDYID to hkp://keys.openpgp.org:80

Thanks to https://gist.github.com/troyfontaine/18c9146295168ee9ca2b30c00bd1b41e for the assist.

2021-06-30