diff --git a/.github/workflows/postgresql-16-ppg-package-pgxs.yml b/.github/workflows/postgresql-16-ppg-package-pgxs.yml index ba671052..ccf758de 100644 --- a/.github/workflows/postgresql-16-ppg-package-pgxs.yml +++ b/.github/workflows/postgresql-16-ppg-package-pgxs.yml @@ -29,7 +29,7 @@ jobs: run: | sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \ zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \ - libcurl4-openssl-dev libhttp-server-simple-perl + libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -64,6 +64,11 @@ jobs: with: path: 'src/pg_tde' + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/pg_tde + - name: Change src owner to postgres run: | sudo chmod o+rx ~ @@ -76,10 +81,18 @@ jobs: sudo make USE_PGXS=1 install working-directory: src/pg_tde + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp + - name: Start pg_tde tests run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.github/workflows/postgresql-16-src-make-macos.yml b/.github/workflows/postgresql-16-src-make-macos.yml deleted file mode 100644 index 2a96d775..00000000 --- a/.github/workflows/postgresql-16-src-make-macos.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: postgresql-16-src-make-macos -on: [pull_request, workflow_dispatch] - -permissions: - contents: read - -jobs: - build: - name: pg-16-src-make-test - runs-on: macos-14 - steps: - - name: Install dependencies - run: | - brew tap hashicorp/tap - brew install gnu-sed icu4c hashicorp/tap/vault - sudo /usr/bin/perl -MCPAN -e 'install HTTP::Server::Simple' - sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' - sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' - - - name: Clone postgres repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: 'postgres/postgres' - ref: 'a81e5516fa4bc53e332cb35eefe231147c0e1749' - path: 'src' - - - name: Clone pg_tde repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: 'src/contrib/pg_tde' - - - name: Create pgsql dir - run: mkdir -p $HOME/pgsql - - - name: Build postgres - run: | - export LDFLAGS="-L/opt/homebrew/opt/openssl/lib" - export CPPFLAGS="-I/opt/homebrew/opt/openssl/include" - export PKG_CONFIG_PATH="/opt/homebrew/opt/icu4c/lib/pkgconfig" - ./configure --with-openssl --enable-tap-tests=no --prefix=$HOME/postgres - make -j 4 - sudo make install - working-directory: src - - - name: Build pg_tde - run: | - ./configure - make -j 4 MAJORVERSION=16 - sudo make install - working-directory: src/contrib/pg_tde - - - name: Start postgresql cluster with pg_tde - run: | - export PATH="$HOME/postgres/bin:$PATH" - initdb -D $HOME/pgsql/data - echo "shared_preload_libraries = 'pg_tde'" >> \ - /$HOME/pgsql/data/postgresql.conf - pg_ctl -D $HOME/pgsql/data -l logfile start - - - name: Test pg_tde - run: | - TV=$(mktemp) - { exec >$TV; vault server -dev; } & - sleep 10 - export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) - echo "Root token: $ROOT_TOKEN" - make installcheck - working-directory: src/contrib/pg_tde - - - name: Report on test fail - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - if: ${{ failure() }} - with: - name: Regressions diff and postgresql log - path: | - src/contrib/pg_tde/regression.diffs - retention-days: 3 - - - name: Run debug commands on failure - if: ${{ failure() }} - run: | - env - pwd diff --git a/.github/workflows/postgresql-16-src-make-ssl11.yml b/.github/workflows/postgresql-16-src-make-ssl11.yml index 2c20d519..d793c513 100644 --- a/.github/workflows/postgresql-16-src-make-ssl11.yml +++ b/.github/workflows/postgresql-16-src-make-ssl11.yml @@ -29,7 +29,7 @@ jobs: libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \ libsystemd-dev gettext tcl-dev libperl-dev pkg-config \ libselinux1-dev python3-dev libhttp-server-simple-perl\ - uuid-dev liblz4-dev libcurl4-openssl-dev + uuid-dev liblz4-dev libcurl4-openssl-dev python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -47,6 +47,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: 'src/contrib/pg_tde' + + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/contrib/pg_tde - name: Create pgsql dir run: mkdir -p /opt/pgsql @@ -74,10 +79,18 @@ jobs: /opt/pgsql/data/postgresql.conf pg_ctl -D /opt/pgsql/data -l logfile start + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp + - name: Test pg_tde run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.github/workflows/postgresql-16-src-make.yml b/.github/workflows/postgresql-16-src-make.yml index 41237c49..2b5c3d06 100644 --- a/.github/workflows/postgresql-16-src-make.yml +++ b/.github/workflows/postgresql-16-src-make.yml @@ -29,7 +29,7 @@ jobs: libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \ libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ llvm-11 llvm-11-dev libselinux1-dev python3-dev \ - uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl + uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -47,6 +47,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: 'src/contrib/pg_tde' + + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/contrib/pg_tde - name: Create pgsql dir run: mkdir -p /opt/pgsql @@ -73,11 +78,19 @@ jobs: echo "shared_preload_libraries = 'pg_tde'" >> \ /opt/pgsql/data/postgresql.conf pg_ctl -D /opt/pgsql/data -l logfile start + + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp - name: Test pg_tde run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.github/workflows/postgresql-16-src-meson-macos.yml b/.github/workflows/postgresql-16-src-meson-macos.yml deleted file mode 100644 index 0513277f..00000000 --- a/.github/workflows/postgresql-16-src-meson-macos.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: postgresql-16-src-meson-macos -on: [pull_request, workflow_dispatch] - -permissions: - contents: read - -jobs: - build: - name: pg-16-src-meson-test - runs-on: macos-14 - steps: - - name: Install dependencies - run: | - brew tap hashicorp/tap - brew install meson gnu-sed icu4c hashicorp/tap/vault - sudo /usr/bin/perl -MCPAN -e 'install HTTP::Server::Simple' - sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' - sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' - - - name: Clone postgres repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: 'postgres/postgres' - ref: 'a81e5516fa4bc53e332cb35eefe231147c0e1749' - path: 'src' - - - name: Clone pg_tde repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: 'src/contrib/pg_tde' - - - name: Include pg_tde in meson build - run: | - echo "subdir('pg_tde')" >> src/contrib/meson.build - - - name: Build postgres - run: | - export PKG_CONFIG_PATH="/opt/homebrew/opt/icu4c/lib/pkgconfig" - meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true - cd build && ninja && ninja install - working-directory: src - - - name: Test pg_tde - run: | - TV=$(mktemp) - { exec >$TV; vault server -dev; } & - sleep 10 - export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) - echo "Root token: $ROOT_TOKEN" - meson test --suite setup -v - meson test --suite pg_tde -v --num-processes 1 - working-directory: src/build - - - name: Report on test fail - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - if: ${{ failure() }} - with: - name: Regressions diff and postgresql log - path: | - src/build/testrun/pg_tde/regress/ - retention-days: 3 - - - name: Run debug commands on failure - if: ${{ failure() }} - run: | - env - pwd diff --git a/.github/workflows/postgresql-16-src-meson.yml b/.github/workflows/postgresql-16-src-meson.yml index b9f6aabf..a2f35eb0 100644 --- a/.github/workflows/postgresql-16-src-meson.yml +++ b/.github/workflows/postgresql-16-src-meson.yml @@ -30,7 +30,7 @@ jobs: libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ llvm-11 llvm-11-dev libselinux1-dev python3-dev \ uuid-dev liblz4-dev meson ninja-build \ - gpg wget libcurl4-openssl-dev libhttp-server-simple-perl + gpg wget libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -48,6 +48,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: 'src/contrib/pg_tde' + + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/contrib/pg_tde - name: Include pg_tde in meson build run: | @@ -58,11 +63,19 @@ jobs: meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true cd build && ninja && ninja install working-directory: src + + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp - name: Test pg_tde run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.github/workflows/postgresql-17-src-make.yml b/.github/workflows/postgresql-17-src-make.yml index 0ae70488..8a99702c 100644 --- a/.github/workflows/postgresql-17-src-make.yml +++ b/.github/workflows/postgresql-17-src-make.yml @@ -29,7 +29,7 @@ jobs: libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \ libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ llvm-11 llvm-11-dev libselinux1-dev python3-dev \ - uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl + uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -47,6 +47,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: 'src/contrib/pg_tde' + + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/contrib/pg_tde - name: Create pgsql dir run: mkdir -p /opt/pgsql @@ -74,10 +79,18 @@ jobs: /opt/pgsql/data/postgresql.conf pg_ctl -D /opt/pgsql/data -l logfile start + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp + - name: Test pg_tde run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.github/workflows/postgresql-17-src-meson-perf.yml b/.github/workflows/postgresql-17-src-meson-perf.yml index 60de4ed6..798f0ce0 100644 --- a/.github/workflows/postgresql-17-src-meson-perf.yml +++ b/.github/workflows/postgresql-17-src-meson-perf.yml @@ -30,7 +30,7 @@ jobs: libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ llvm-11 llvm-11-dev libselinux1-dev python3-dev \ uuid-dev liblz4-dev meson ninja-build \ - sysbench libcurl4-openssl-dev libhttp-server-simple-perl + sysbench libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -48,6 +48,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: 'src/contrib/pg_tde' + + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/contrib/pg_tde - name: Build postgres run: | @@ -55,10 +60,18 @@ jobs: cd build && ninja && ninja install working-directory: src + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp + - name: Test pg_tde run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.github/workflows/postgresql-17-src-meson.yml b/.github/workflows/postgresql-17-src-meson.yml index 9c7dfee6..fdb5daf8 100644 --- a/.github/workflows/postgresql-17-src-meson.yml +++ b/.github/workflows/postgresql-17-src-meson.yml @@ -30,7 +30,7 @@ jobs: libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ llvm-11 llvm-11-dev libselinux1-dev python3-dev \ uuid-dev liblz4-dev meson ninja-build \ - gpg wget libcurl4-openssl-dev libhttp-server-simple-perl + gpg wget libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -49,20 +49,32 @@ jobs: with: path: 'src/contrib/pg_tde' + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/contrib/pg_tde + - name: Build postgres run: | meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true cd build && ninja && ninja install working-directory: src + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp + - name: Test pg_tde run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" - meson test --suite setup -v meson test --suite pg_tde -v --num-processes 1 working-directory: src/build diff --git a/.github/workflows/postgresql-pgdg-package-pgxs.yml b/.github/workflows/postgresql-pgdg-package-pgxs.yml index 81b7b9f8..d515c9b2 100644 --- a/.github/workflows/postgresql-pgdg-package-pgxs.yml +++ b/.github/workflows/postgresql-pgdg-package-pgxs.yml @@ -32,7 +32,7 @@ jobs: run: | sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \ zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \ - libcurl4-openssl-dev libhttp-server-simple-perl + libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -57,6 +57,11 @@ jobs: with: path: 'src/pg_tde' + - name: Checkout libkmip submodule + run: | + git submodule update --init --recursive + working-directory: src/pg_tde + - name: Change src owner to postgres run: | sudo chmod o+rx ~ @@ -71,12 +76,20 @@ jobs: sudo make USE_PGXS=1 MAJORVERSION=$POSTGRESQL_VERSION install working-directory: src/pg_tde + - name: Setup kmip 1 + run: | + wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py + python3 create_certificates.py + cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem + working-directory: /tmp + - name: Start pg_tde tests env: POSTGRESQL_VERSION: ${{ matrix.postgresql-version }} run: | TV=$(mktemp) { exec >$TV; vault server -dev; } & + pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log & sleep 10 export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) echo "Root token: $ROOT_TOKEN" diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..ea8f23fc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/libkmip"] + path = src/libkmip + url = https://github.com/Percona-Lab/libkmip.git diff --git a/Makefile.in b/Makefile.in index a11ef2d8..77055a2d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -46,6 +46,8 @@ src/transam/pg_tde_xact_handler.o \ src/keyring/keyring_curl.o \ src/keyring/keyring_file.o \ src/keyring/keyring_vault.o \ +src/keyring/keyring_kmip.o \ +src/keyring/keyring_kmip_ereport.o \ src/keyring/keyring_api.o \ src/catalog/tde_global_space.o \ src/catalog/tde_keyring.o \ @@ -56,19 +58,23 @@ src/common/pg_tde_utils.o \ src/smgr/pg_tde_smgr.o \ src/pg_tde_defs.o \ src/pg_tde_event_capture.o \ -src/pg_tde.o +src/pg_tde.o \ +src/libkmip/libkmip/src/kmip.o \ +src/libkmip/libkmip/src/kmip_bio.o \ +src/libkmip/libkmip/src/kmip_locate.o \ +src/libkmip/libkmip/src/kmip_memset.o override PG_CPPFLAGS += @tde_CPPFLAGS@ ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) -override PG_CPPFLAGS += -I$(CURDIR)/src/include -I$(CURDIR)/src$(MAJORVERSION)/include +override PG_CPPFLAGS += -I$(CURDIR)/src/include -I$(CURDIR)/src/libkmip/libkmip/include -I$(CURDIR)/src$(MAJORVERSION)/include include $(PGXS) else subdir = contrib/pg_tde top_builddir = ../.. -override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include -I$(top_srcdir)/$(subdir)/src$(MAJORVERSION)/include +override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include -I$(top_srcdir)/$(subdir)/src/libkmip/libkmip/include -I$(top_srcdir)/$(subdir)/src$(MAJORVERSION)/include include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif diff --git a/expected/kmip_test.out b/expected/kmip_test.out new file mode 100644 index 00000000..bf9d0789 --- /dev/null +++ b/expected/kmip_test.out @@ -0,0 +1,33 @@ +\set tde_am tde_heap +\i sql/kmip_test.inc +CREATE EXTENSION pg_tde; +SELECT pg_tde_add_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); + pg_tde_add_key_provider_kmip +------------------------------ + 1 +(1 row) + +SELECT pg_tde_set_principal_key('kmip-principal-key','kmip-prov'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +CREATE TABLE test_enc( + id SERIAL, + k INTEGER DEFAULT '0' NOT NULL, + PRIMARY KEY (id) + ) USING :tde_am; +INSERT INTO test_enc (k) VALUES (1); +INSERT INTO test_enc (k) VALUES (2); +INSERT INTO test_enc (k) VALUES (3); +SELECT * from test_enc; + id | k +----+--- + 1 | 1 + 2 | 2 + 3 | 3 +(3 rows) + +DROP TABLE test_enc; +DROP EXTENSION pg_tde; diff --git a/expected/kmip_test_basic.out b/expected/kmip_test_basic.out new file mode 100644 index 00000000..c1074a26 --- /dev/null +++ b/expected/kmip_test_basic.out @@ -0,0 +1,33 @@ +\set tde_am tde_heap_basic +\i sql/kmip_test.inc +CREATE EXTENSION pg_tde; +SELECT pg_tde_add_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); + pg_tde_add_key_provider_kmip +------------------------------ + 1 +(1 row) + +SELECT pg_tde_set_principal_key('kmip-principal-key','kmip-prov'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +CREATE TABLE test_enc( + id SERIAL, + k INTEGER DEFAULT '0' NOT NULL, + PRIMARY KEY (id) + ) USING :tde_am; +INSERT INTO test_enc (k) VALUES (1); +INSERT INTO test_enc (k) VALUES (2); +INSERT INTO test_enc (k) VALUES (3); +SELECT * from test_enc; + id | k +----+--- + 1 | 1 + 2 | 2 + 3 | 3 +(3 rows) + +DROP TABLE test_enc; +DROP EXTENSION pg_tde; diff --git a/kmip-server.conf b/kmip-server.conf new file mode 100644 index 00000000..7644e4b5 --- /dev/null +++ b/kmip-server.conf @@ -0,0 +1,15 @@ +[server] +hostname=127.0.0.1 +port=5696 +certificate_path=/tmp/server_certificate.pem +key_path=/tmp/server_key.pem +ca_path=/tmp/root_certificate.pem +auth_suite=TLS1.2 +policy_path=/path/to/policy/file +enable_tls_client_auth=True +tls_cipher_suites= + TLS_RSA_WITH_AES_128_CBC_SHA256 + TLS_RSA_WITH_AES_256_CBC_SHA256 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 +logging_level=DEBUG +database_path=/tmp/pykmip.db diff --git a/meson.build b/meson.build index 06e971c8..c1ceb10f 100644 --- a/meson.build +++ b/meson.build @@ -40,6 +40,8 @@ pg_tde_sources = files( 'src/keyring/keyring_curl.c', 'src/keyring/keyring_file.c', 'src/keyring/keyring_vault.c', + 'src/keyring/keyring_kmip.c', + 'src/keyring/keyring_kmip_ereport.c', 'src/keyring/keyring_api.c', 'src/smgr/pg_tde_smgr.c', @@ -53,9 +55,14 @@ pg_tde_sources = files( 'src/pg_tde_defs.c', 'src/pg_tde.c', 'src/pg_tde_event_capture.c', + + 'src/libkmip/libkmip/src/kmip.c', + 'src/libkmip/libkmip/src/kmip_bio.c', + 'src/libkmip/libkmip/src/kmip_locate.c', + 'src/libkmip/libkmip/src/kmip_memset.c', ) -incdir = include_directories(src_version / 'include', 'src/include', '.') +incdir = include_directories(src_version / 'include', 'src/include', '.', 'src/libkmip/libkmip/include/') deps_update = {'dependencies': contrib_mod_args.get('dependencies') + [curldep]} @@ -98,6 +105,7 @@ sql_tests = [ 'insert_update_delete_basic', 'tablespace_basic', 'vault_v2_test_basic', + 'kmip_test_basic', 'alter_index_basic', 'merge_join_basic', ] @@ -130,6 +138,7 @@ if get_variable('percona_ext', false) 'insert_update_delete', 'tablespace', 'vault_v2_test', + 'kmip_test', 'alter_index', 'merge_join', ] diff --git a/pg_tde--1.0.sql b/pg_tde--1.0.sql index 727fac5a..93cb6106 100644 --- a/pg_tde--1.0.sql +++ b/pg_tde--1.0.sql @@ -75,6 +75,41 @@ AS $$ $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_kmip(provider_name VARCHAR(128), + kmip_host TEXT, + kmip_port INT, + kmip_ca_path TEXT, + kmip_cert_path TEXT) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_kmip_keyring_provider_options function. + SELECT pg_tde_add_key_provider('kmip', provider_name, + json_object('type' VALUE 'kmip', + 'host' VALUE COALESCE(kmip_host,''), + 'port' VALUE kmip_port, + 'caPath' VALUE COALESCE(kmip_ca_path,''), + 'certPath' VALUE COALESCE(kmip_cert_path,''))); +$$ +LANGUAGE SQL; +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_kmip(provider_name VARCHAR(128), + kmip_host JSON, + kmip_port JSON, + kmip_ca_path JSON, + kmip_cert_path JSON) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_kmip_keyring_provider_options function. + SELECT pg_tde_add_key_provider('kmip', provider_name, + json_object('type' VALUE 'kmip', + 'host' VALUE kmip_host, + 'port' VALUE kmip_port, + 'caPath' VALUE kmip_ca_path, + 'certPath' VALUE kmip_cert_path)); +$$ +LANGUAGE SQL; + CREATE FUNCTION pg_tde_list_all_key_providers (OUT id INT, OUT provider_name VARCHAR(128), @@ -152,6 +187,43 @@ AS $$ $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_kmip(PG_TDE_GLOBAL, + provider_name VARCHAR(128), + kmip_host TEXT, + kmip_port INT, + kmip_ca_path TEXT, + kmip_cert_path TEXT) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_kmip_keyring_provider_options function. + SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'kmip', provider_name, + json_object('type' VALUE 'kmip', + 'host' VALUE COALESCE(kmip_host,''), + 'port' VALUE kmip_port, + 'caPath' VALUE COALESCE(kmip_ca_path,''), + 'certPath' VALUE COALESCE(vault_cert_path,''))); +$$ +LANGUAGE SQL; +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_kmip(PG_TDE_GLOBAL, + provider_name VARCHAR(128), + kmip_host JSON, + kmip_port JSON, + kmip_ca_path JSON, + kmip_cert_path JSON) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_kmip_keyring_provider_options function. + SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'vault-v2', provider_name, + json_object('type' VALUE 'vault-v2', + 'host' VALUE kmip_host, + 'port' VALUE kmip_port, + 'caPath' VALUE kmip_ca_path, + 'certPath' VALUE kmip_cert_path)); +$$ +LANGUAGE SQL; + -- Table access method CREATE FUNCTION pg_tdeam_basic_handler(internal) RETURNS table_am_handler diff --git a/pykmip-server.conf b/pykmip-server.conf new file mode 100644 index 00000000..7644e4b5 --- /dev/null +++ b/pykmip-server.conf @@ -0,0 +1,15 @@ +[server] +hostname=127.0.0.1 +port=5696 +certificate_path=/tmp/server_certificate.pem +key_path=/tmp/server_key.pem +ca_path=/tmp/root_certificate.pem +auth_suite=TLS1.2 +policy_path=/path/to/policy/file +enable_tls_client_auth=True +tls_cipher_suites= + TLS_RSA_WITH_AES_128_CBC_SHA256 + TLS_RSA_WITH_AES_256_CBC_SHA256 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 +logging_level=DEBUG +database_path=/tmp/pykmip.db diff --git a/sql/kmip_test.inc b/sql/kmip_test.inc new file mode 100644 index 00000000..e748b862 --- /dev/null +++ b/sql/kmip_test.inc @@ -0,0 +1,20 @@ +CREATE EXTENSION pg_tde; + +SELECT pg_tde_add_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); +SELECT pg_tde_set_principal_key('kmip-principal-key','kmip-prov'); + +CREATE TABLE test_enc( + id SERIAL, + k INTEGER DEFAULT '0' NOT NULL, + PRIMARY KEY (id) + ) USING :tde_am; + +INSERT INTO test_enc (k) VALUES (1); +INSERT INTO test_enc (k) VALUES (2); +INSERT INTO test_enc (k) VALUES (3); + +SELECT * from test_enc; + +DROP TABLE test_enc; + +DROP EXTENSION pg_tde; diff --git a/sql/kmip_test.sql b/sql/kmip_test.sql new file mode 100644 index 00000000..4dffe634 --- /dev/null +++ b/sql/kmip_test.sql @@ -0,0 +1,2 @@ +\set tde_am tde_heap +\i sql/kmip_test.inc \ No newline at end of file diff --git a/sql/kmip_test_basic.sql b/sql/kmip_test_basic.sql new file mode 100644 index 00000000..41b37db5 --- /dev/null +++ b/sql/kmip_test_basic.sql @@ -0,0 +1,2 @@ +\set tde_am tde_heap_basic +\i sql/kmip_test.inc diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index 95a9c669..bd187b8b 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -52,6 +52,7 @@ typedef enum ProviderScanType static FileKeyring *load_file_keyring_provider_options(char *keyring_options); static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, char *keyring_options); static VaultV2Keyring *load_vaultV2_keyring_provider_options(char *keyring_options); +static KmipKeyring *load_kmip_keyring_provider_options(char *keyring_options); static void debug_print_kerying(GenericKeyring *keyring); static GenericKeyring *load_keyring_provider_from_record(KeyringProvideRecord *provider); static inline void get_keyring_infofile_path(char *resPath, Oid dbOid); @@ -149,6 +150,8 @@ get_keyring_provider_from_typename(char *provider_type) return FILE_KEY_PROVIDER; if (strcmp(VAULTV2_KEYRING_TYPE, provider_type) == 0) return VAULT_V2_KEY_PROVIDER; + if (strcmp(KMIP_KEYRING_TYPE, provider_type) == 0) + return KMIP_KEY_PROVIDER; return UNKNOWN_KEY_PROVIDER; } @@ -161,6 +164,8 @@ get_keyring_provider_typename(ProviderType p_type) return FILE_KEYRING_TYPE; case VAULT_V2_KEY_PROVIDER: return VAULTV2_KEYRING_TYPE; + case KMIP_KEY_PROVIDER: + return KMIP_KEYRING_TYPE; default: break; } @@ -559,6 +564,9 @@ load_keyring_provider_options(ProviderType provider_type, char *keyring_options) case VAULT_V2_KEY_PROVIDER: return (GenericKeyring *) load_vaultV2_keyring_provider_options(keyring_options); break; + case KMIP_KEY_PROVIDER: + return (GenericKeyring *)load_kmip_keyring_provider_options(keyring_options); + break; default: break; } @@ -618,6 +626,35 @@ load_vaultV2_keyring_provider_options(char *keyring_options) return vaultV2_keyring; } +static KmipKeyring * +load_kmip_keyring_provider_options(char *keyring_options) +{ + KmipKeyring *kmip_keyring = palloc0(sizeof(KmipKeyring)); + + kmip_keyring->keyring.type = KMIP_KEY_PROVIDER; + + if (!ParseKeyringJSONOptions(KMIP_KEY_PROVIDER, kmip_keyring, + keyring_options, strlen(keyring_options))) + { + return NULL; + } + + if (strlen(kmip_keyring->kmip_host) == 0 || + strlen(kmip_keyring->kmip_ca_path) == 0 || + strlen(kmip_keyring->kmip_cert_path) == 0) + { + ereport(WARNING, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing in the keyring options:%s%s%s", + *(kmip_keyring->kmip_host) ? "" : " kmip_host", + *(kmip_keyring->kmip_ca_path) ? "" : " kmip_ca_path", + *(kmip_keyring->kmip_cert_path) ? "" : " kmip_cert_path"))); + return NULL; + } + + return kmip_keyring; +} + static void debug_print_kerying(GenericKeyring *keyring) { @@ -637,6 +674,12 @@ debug_print_kerying(GenericKeyring *keyring) elog(debug_level, "Vault Keyring Mount Path: %s", ((VaultV2Keyring *) keyring)->vault_mount_path); elog(debug_level, "Vault Keyring CA Path: %s", ((VaultV2Keyring *) keyring)->vault_ca_path); break; + case KMIP_KEY_PROVIDER: + elog(debug_level, "KMIP Keyring Host: %s", ((KmipKeyring *)keyring)->kmip_host); + elog(debug_level, "KMIP Keyring Port: %s", ((KmipKeyring *)keyring)->kmip_port); + elog(debug_level, "KMIP Keyring CA Path: %s", ((KmipKeyring *)keyring)->kmip_ca_path); + elog(debug_level, "KMIP Keyring Cert Path: %s", ((KmipKeyring *)keyring)->kmip_cert_path); + break; case UNKNOWN_KEY_PROVIDER: elog(debug_level, "Unknown Keyring "); break; diff --git a/src/catalog/tde_keyring_parse_opts.c b/src/catalog/tde_keyring_parse_opts.c index e9334fa8..499f83f3 100644 --- a/src/catalog/tde_keyring_parse_opts.c +++ b/src/catalog/tde_keyring_parse_opts.c @@ -69,6 +69,11 @@ typedef enum JsonKeyringField JK_VAULT_MOUNT_PATH, JK_VAULT_CA_PATH, + JK_KMIP_HOST, + JK_KMIP_PORT, + JK_KMIP_CA_PATH, + JK_KMIP_CERT_PATH, + /* must be the last */ JK_FIELDS_TOTAL } JsonKeyringField; @@ -89,6 +94,11 @@ static const char *JK_FIELD_NAMES[JK_FIELDS_TOTAL] = { [JK_VAULT_URL] = "url", [JK_VAULT_MOUNT_PATH] = "mountPath", [JK_VAULT_CA_PATH] = "caPath", + + [JK_KMIP_HOST] = "host", + [JK_KMIP_PORT] = "port", + [JK_KMIP_CA_PATH] = "caPath", + [JK_KMIP_CERT_PATH] = "certPath", }; #define MAX_JSON_DEPTH 64 @@ -331,6 +341,26 @@ json_kring_object_field_start(void *state, char *fname, bool isnull) elog(DEBUG1, "parse json keyring config: unexpected field %s", fname); } break; + + case KMIP_KEY_PROVIDER: + if (strcmp(fname, JK_FIELD_NAMES[JK_KMIP_HOST]) == 0) + *field = JK_KMIP_HOST; + else if (strcmp(fname, JK_FIELD_NAMES[JK_KMIP_PORT]) == 0) + *field = JK_KMIP_PORT; + else if (strcmp(fname, JK_FIELD_NAMES[JK_KMIP_CA_PATH]) == 0) + *field = JK_KMIP_CA_PATH; + else if (strcmp(fname, JK_FIELD_NAMES[JK_KMIP_CERT_PATH]) == 0) + *field = JK_KMIP_CERT_PATH; + else + { + *field = JK_FIELD_UNKNOWN; + elog(DEBUG1, "parse json keyring config: unexpected field %s", fname); + } + break; + + case UNKNOWN_KEY_PROVIDER: + Assert(0); + break; } break; @@ -368,6 +398,7 @@ json_kring_assign_scalar(JsonKeyringState *parse, JsonKeyringField field, char * { VaultV2Keyring *vault = parse->provider_opts; FileKeyring *file = parse->provider_opts; + KmipKeyring *kmip = parse->provider_opts; switch (field) { @@ -402,6 +433,19 @@ json_kring_assign_scalar(JsonKeyringState *parse, JsonKeyringField field, char * strncpy(vault->vault_ca_path, value, sizeof(vault->vault_ca_path)); break; + case JK_KMIP_HOST: + strncpy(kmip->kmip_host, value, sizeof(kmip->kmip_host)); + break; + case JK_KMIP_PORT: + strncpy(kmip->kmip_port, value, sizeof(kmip->kmip_port)); + break; + case JK_KMIP_CA_PATH: + strncpy(kmip->kmip_ca_path, value, sizeof(kmip->kmip_ca_path)); + break; + case JK_KMIP_CERT_PATH: + strncpy(kmip->kmip_cert_path, value, sizeof(kmip->kmip_cert_path)); + break; + default: elog(DEBUG1, "json keyring: unexpected scalar field %d", field); Assert(0); diff --git a/src/include/access/pg_tde_xlog_encrypt_fe.h b/src/include/access/pg_tde_xlog_encrypt_fe.h index eca9bc8d..4717afb7 100644 --- a/src/include/access/pg_tde_xlog_encrypt_fe.h +++ b/src/include/access/pg_tde_xlog_encrypt_fe.h @@ -15,15 +15,17 @@ #include "encryption/enc_aes.h" #include "keyring/keyring_file.h" #include "keyring/keyring_vault.h" +#include "keyring/keyring_kmip.h" /* Frontend has to call it needs to read an encrypted XLog */ #define TDE_XLOG_INIT(kring_dir) \ AesInit(); \ InstallFileKeyring(); \ InstallVaultV2Keyring(); \ + InstallKmipKeyring(); \ TDEInitGlobalKeys(kring_dir); \ TDEXLogSmgrInit() #endif /* PERCONA_EXT */ -#endif /* PG_TDE_XLOGENCRYPT_FE_H */ +#endif /* PG_TDE_XLOGENCRYPT_FE_H */ diff --git a/src/include/catalog/keyring_min.h b/src/include/catalog/keyring_min.h new file mode 100644 index 00000000..00c1c420 --- /dev/null +++ b/src/include/catalog/keyring_min.h @@ -0,0 +1,102 @@ + +#ifndef KEYRING_MIN_H_ +#define KEYRING_MIN_H_ + +#include "pg_config_manual.h" + +/* This is a minimal header that doesn't depend on postgres headers to avoid a type conflict with libkmip */ + +typedef unsigned int Oid; + +#define MAX_PROVIDER_NAME_LEN 128 /* pg_tde_key_provider's provider_name size*/ +#define MAX_VAULT_V2_KEY_LEN 128 /* From hashi corp docs */ +#define MAX_KEYRING_OPTION_LEN 1024 +typedef enum ProviderType +{ + UNKNOWN_KEY_PROVIDER, + FILE_KEY_PROVIDER, + VAULT_V2_KEY_PROVIDER, + KMIP_KEY_PROVIDER, +} ProviderType; + +#define TDE_KEY_NAME_LEN 256 +#define MAX_KEY_DATA_SIZE 32 /* maximum 256 bit encryption */ +#define INTERNAL_KEY_LEN 16 + +typedef struct keyName +{ + char name[TDE_KEY_NAME_LEN]; +} keyName; + +typedef struct keyData +{ + unsigned char data[MAX_KEY_DATA_SIZE]; + unsigned len; +} keyData; + +typedef struct keyInfo +{ + keyName name; + keyData data; +} keyInfo; + +typedef enum KeyringReturnCodes +{ + KEYRING_CODE_SUCCESS = 0, + KEYRING_CODE_INVALID_PROVIDER, + KEYRING_CODE_RESOURCE_NOT_AVAILABLE, + KEYRING_CODE_RESOURCE_NOT_ACCESSABLE, + KEYRING_CODE_INVALID_OPERATION, + KEYRING_CODE_INVALID_RESPONSE, + KEYRING_CODE_INVALID_KEY_SIZE, + KEYRING_CODE_DATA_CORRUPTED +} KeyringReturnCodes; + +/* Base type for all keyring */ +typedef struct GenericKeyring +{ + ProviderType type; /* Must be the first field */ + Oid key_id; + char provider_name[MAX_PROVIDER_NAME_LEN]; + char options[MAX_KEYRING_OPTION_LEN]; /* User provided options string*/ +} GenericKeyring; + +typedef struct TDEKeyringRoutine +{ + keyInfo *(*keyring_get_key) (GenericKeyring *keyring, const char *key_name, bool throw_error, KeyringReturnCodes * returnCode); + KeyringReturnCodes(*keyring_store_key) (GenericKeyring *keyring, keyInfo *key, bool throw_error); +} TDEKeyringRoutine; + +/* + * Keyring type name must be in sync with catalog table + * defination in pg_tde--1.0 SQL + */ +#define FILE_KEYRING_TYPE "file" +#define VAULTV2_KEYRING_TYPE "vault-v2" +#define KMIP_KEYRING_TYPE "kmip" + +typedef struct FileKeyring +{ + GenericKeyring keyring; /* Must be the first field */ + char file_name[MAXPGPATH]; +} FileKeyring; + +typedef struct VaultV2Keyring +{ + GenericKeyring keyring; /* Must be the first field */ + char vault_token[MAX_VAULT_V2_KEY_LEN]; + char vault_url[MAXPGPATH]; + char vault_ca_path[MAXPGPATH]; + char vault_mount_path[MAXPGPATH]; +} VaultV2Keyring; + +typedef struct KmipKeyring +{ + GenericKeyring keyring; /* Must be the first field */ + char kmip_host[MAXPGPATH]; + char kmip_port[32]; + char kmip_ca_path[MAXPGPATH]; + char kmip_cert_path[MAXPGPATH]; +} KmipKeyring; + +#endif \ No newline at end of file diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index 6249ba47..48d9b131 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -12,50 +12,10 @@ #include "postgres.h" #include "nodes/pg_list.h" +#include "catalog/keyring_min.h" #define PG_TDE_NAMESPACE_NAME "percona_tde" #define PG_TDE_KEY_PROVIDER_CAT_NAME "pg_tde_key_provider" -/* - * Keyring type name must be in sync with catalog table - * defination in pg_tde--1.0 SQL - */ -#define FILE_KEYRING_TYPE "file" -#define VAULTV2_KEYRING_TYPE "vault-v2" - -#define MAX_PROVIDER_NAME_LEN 128 /* pg_tde_key_provider's provider_name - * size */ -#define MAX_VAULT_V2_KEY_LEN 128 /* From hashi corp docs */ -#define MAX_KEYRING_OPTION_LEN 1024 -typedef enum ProviderType -{ - UNKNOWN_KEY_PROVIDER, - FILE_KEY_PROVIDER, - VAULT_V2_KEY_PROVIDER, -} ProviderType; - -/* Base type for all keyring */ -typedef struct GenericKeyring -{ - ProviderType type; /* Must be the first field */ - Oid key_id; - char provider_name[MAX_PROVIDER_NAME_LEN]; - char options[MAX_KEYRING_OPTION_LEN]; /* User provided options string */ -} GenericKeyring; - -typedef struct FileKeyring -{ - GenericKeyring keyring; /* Must be the first field */ - char file_name[MAXPGPATH]; -} FileKeyring; - -typedef struct VaultV2Keyring -{ - GenericKeyring keyring; /* Must be the first field */ - char vault_token[MAX_VAULT_V2_KEY_LEN]; - char vault_url[MAXPGPATH]; - char vault_ca_path[MAXPGPATH]; - char vault_mount_path[MAXPGPATH]; -} VaultV2Keyring; /* This record goes into key provider info file */ typedef struct KeyringProvideRecord diff --git a/src/include/keyring/keyring_api.h b/src/include/keyring/keyring_api.h index 147302de..25f08be2 100644 --- a/src/include/keyring/keyring_api.h +++ b/src/include/keyring/keyring_api.h @@ -10,45 +10,7 @@ #define KEYRING_API_H #include "catalog/tde_keyring.h" - -#define TDE_KEY_NAME_LEN 256 -#define MAX_KEY_DATA_SIZE 32 /* maximum 256 bit encryption */ -#define INTERNAL_KEY_LEN 16 - -typedef struct keyName -{ - char name[TDE_KEY_NAME_LEN]; -} keyName; - -typedef struct keyData -{ - unsigned char data[MAX_KEY_DATA_SIZE]; - unsigned len; -} keyData; - -typedef struct keyInfo -{ - keyName name; - keyData data; -} keyInfo; - -typedef enum KeyringReturnCodes -{ - KEYRING_CODE_SUCCESS = 0, - KEYRING_CODE_INVALID_PROVIDER, - KEYRING_CODE_RESOURCE_NOT_AVAILABLE, - KEYRING_CODE_RESOURCE_NOT_ACCESSABLE, - KEYRING_CODE_INVALID_OPERATION, - KEYRING_CODE_INVALID_RESPONSE, - KEYRING_CODE_INVALID_KEY_SIZE, - KEYRING_CODE_DATA_CORRUPTED -} KeyringReturnCodes; - -typedef struct TDEKeyringRoutine -{ - keyInfo *(*keyring_get_key) (GenericKeyring *keyring, const char *key_name, bool throw_error, KeyringReturnCodes * returnCode); - KeyringReturnCodes(*keyring_store_key) (GenericKeyring *keyring, keyInfo *key, bool throw_error); -} TDEKeyringRoutine; +#include "catalog/keyring_min.h" extern bool RegisterKeyProvider(const TDEKeyringRoutine *routine, ProviderType type); diff --git a/src/include/keyring/keyring_kmip.h b/src/include/keyring/keyring_kmip.h new file mode 100644 index 00000000..f168202c --- /dev/null +++ b/src/include/keyring/keyring_kmip.h @@ -0,0 +1,19 @@ +/*------------------------------------------------------------------------- + * + * keyring_vault.h + * KMIP based keyring provider + * + * IDENTIFICATION + * src/include/keyring/keyring_kmip.h + * + *------------------------------------------------------------------------- + */ + +#ifndef KEYRING_KMIP_H +#define KEYRING_KMIP_H + +extern bool InstallKmipKeyring(void); + +void kmip_ereport(bool throw_error, const char *msg, int errCode); + +#endif // KEYRING_KMIP_H diff --git a/src/keyring/keyring_kmip.c b/src/keyring/keyring_kmip.c new file mode 100644 index 00000000..e1d570b6 --- /dev/null +++ b/src/keyring/keyring_kmip.c @@ -0,0 +1,253 @@ +/*------------------------------------------------------------------------- + * + * keyring_kmip.c + * KMIP based keyring provider + * + * IDENTIFICATION + * contrib/pg_tde/src/keyring/keyring_kmip.c + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +/* The KMIP headers and Postgres headers conflict. + We can't include most postgres headers here, instead just copy required declarations. +*/ + +#define bool int +#define true 1 +#define false 0 + +#include "keyring/keyring_kmip.h" +#include "catalog/keyring_min.h" + +extern bool RegisterKeyProvider(const TDEKeyringRoutine *routine, ProviderType type); + +static KeyringReturnCodes set_key_by_name(GenericKeyring *keyring, keyInfo *key, bool throw_error); +static keyInfo *get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, KeyringReturnCodes *return_code); + +const TDEKeyringRoutine keyringKmipRoutine = { + .keyring_get_key = get_key_by_name, + .keyring_store_key = set_key_by_name}; + +bool InstallKmipKeyring(void) +{ + return RegisterKeyProvider(&keyringKmipRoutine, KMIP_KEY_PROVIDER); +} + +typedef struct KmipCtx +{ + SSL_CTX *ssl; + BIO *bio; +} KmipCtx; + +static bool kmipSslConnect(KmipCtx *ctx, KmipKeyring *kmip_keyring, bool throw_error) +{ + SSL *ssl = NULL; + ctx->ssl = SSL_CTX_new(SSLv23_method()); + + if (SSL_CTX_use_certificate_file(ctx->ssl, kmip_keyring->kmip_cert_path, SSL_FILETYPE_PEM) != 1) + { + kmip_ereport(throw_error, "SSL error: Loading the client certificate failed", 0); + SSL_CTX_free(ctx->ssl); + return false; + } + + if (SSL_CTX_use_PrivateKey_file(ctx->ssl, kmip_keyring->kmip_cert_path, SSL_FILETYPE_PEM) != 1) + { + SSL_CTX_free(ctx->ssl); + kmip_ereport(throw_error, "SSL error: Loading the client key failed", 0); + return false; + } + + if (SSL_CTX_load_verify_locations(ctx->ssl, kmip_keyring->kmip_ca_path, NULL) != 1) + { + SSL_CTX_free(ctx->ssl); + kmip_ereport(throw_error, "SSL error: Loading the CA certificate failed", 0); + return false; + } + + ctx->bio = BIO_new_ssl_connect(ctx->ssl); + if (ctx->bio == NULL) + { + SSL_CTX_free(ctx->ssl); + kmip_ereport(throw_error, "SSL error: BIO_new_ssl_connect failed", 0); + return false; + } + + BIO_get_ssl(ctx->bio, &ssl); + SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY); + BIO_set_conn_hostname(ctx->bio, kmip_keyring->kmip_host); + BIO_set_conn_port(ctx->bio, kmip_keyring->kmip_port); + if (BIO_do_connect(ctx->bio) != 1) + { + BIO_free_all(ctx->bio); + SSL_CTX_free(ctx->ssl); + kmip_ereport(throw_error, "SSL error: BIO_do_connect failed", 0); + return false; + } + + return true; +} + +static KeyringReturnCodes +set_key_by_name(GenericKeyring *keyring, keyInfo *key, bool throw_error) +{ + KmipCtx ctx; + KmipKeyring *kmip_keyring = (KmipKeyring *)keyring; + int result; + int id_max_len = 64; + char *idp = NULL; + + Attribute a[4]; + enum cryptographic_algorithm algorithm = KMIP_CRYPTOALG_AES; + int32 length = key->data.len * 8; + int32 mask = KMIP_CRYPTOMASK_ENCRYPT | KMIP_CRYPTOMASK_DECRYPT; + Name ts; + TextString ts2 = {0, 0}; + TemplateAttribute ta = {0}; + + if (!kmipSslConnect(&ctx, kmip_keyring, throw_error)) + { + return KEYRING_CODE_INVALID_RESPONSE; + } + + for (int i = 0; i < 4; i++) + { + kmip_init_attribute(&a[i]); + } + + a[0].type = KMIP_ATTR_CRYPTOGRAPHIC_ALGORITHM; + a[0].value = &algorithm; + + a[1].type = KMIP_ATTR_CRYPTOGRAPHIC_LENGTH; + a[1].value = &length; + + a[2].type = KMIP_ATTR_CRYPTOGRAPHIC_USAGE_MASK; + a[2].value = &mask; + + ts2.value = key->name.name; + ts2.size = kmip_strnlen_s(key->name.name, 250); + ts.value = &ts2; + ts.type = KMIP_NAME_UNINTERPRETED_TEXT_STRING; + a[3].type = KMIP_ATTR_NAME; + a[3].value = &ts; + + ta.attributes = a; + ta.attribute_count = ARRAY_LENGTH(a); + + result = kmip_bio_register_symmetric_key(ctx.bio, &ta, (char *)key->data.data, key->data.len, &idp, &id_max_len); + + BIO_free_all(ctx.bio); + SSL_CTX_free(ctx.ssl); + + if (result != 0) + { + kmip_ereport(throw_error, "KMIP server reported error on register symmetric key: %i", result); + return KEYRING_CODE_INVALID_RESPONSE; + } + + return KEYRING_CODE_SUCCESS; +} + +void * + palloc(size_t); + +void pfree(void *); + +static keyInfo *get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, KeyringReturnCodes *return_code) +{ + keyInfo *key = NULL; + KmipKeyring *kmip_keyring = (KmipKeyring *)keyring; + char *id = 0; + KmipCtx ctx; + + *return_code = KEYRING_CODE_SUCCESS; + + if (!kmipSslConnect(&ctx, kmip_keyring, throw_error)) + { + return NULL; + } + + // 1. locate key + + { + int upto = 0; + int all = 1; + LocateResponse locate_result; + Name ts; + TextString ts2 = {0, 0}; + Attribute a[3]; + enum object_type loctype = KMIP_OBJTYPE_SYMMETRIC_KEY; + + for (int i = 0; i < 3; i++) + { + kmip_init_attribute(&a[i]); + } + + a[0].type = KMIP_ATTR_OBJECT_TYPE; + a[0].value = &loctype; + + ts2.value = (char *)key_name; + ts2.size = kmip_strnlen_s(key_name, 250); + ts.value = &ts2; + ts.type = KMIP_NAME_UNINTERPRETED_TEXT_STRING; + a[1].type = KMIP_ATTR_NAME; + a[1].value = &ts; + + // 16 is hard coded: seems like the most vault supports? + int result = kmip_bio_locate(ctx.bio, a, 2, &locate_result, 16, upto); + + if (result != 0) + { + *return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE; + BIO_free_all(ctx.bio); + SSL_CTX_free(ctx.ssl); + return NULL; + } + + if (locate_result.ids_size != 1) + { + kmip_ereport(throw_error, "KMIP server contains multiple results for key, ignoring", 0); + *return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE; + BIO_free_all(ctx.bio); + SSL_CTX_free(ctx.ssl); + return NULL; + } + + id = locate_result.ids[0]; + } + + // 2. get key + + key = palloc(sizeof(keyInfo)); + + { + + char *keyp = NULL; + int result = kmip_bio_get_symmetric_key(ctx.bio, id, strlen(id), &keyp, (int *)&key->data.len); + + if (result != 0) + { + kmip_ereport(throw_error, "KMIP server LOCATEd key, but GET failed with %i", result); + *return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE; + pfree(key); + BIO_free_all(ctx.bio); + SSL_CTX_free(ctx.ssl); + return NULL; + } + + strncpy((char *)key->data.data, keyp, MAX_KEY_DATA_SIZE); + free(keyp); + } + + BIO_free_all(ctx.bio); + SSL_CTX_free(ctx.ssl); + + return key; +} diff --git a/src/keyring/keyring_kmip_ereport.c b/src/keyring/keyring_kmip_ereport.c new file mode 100644 index 00000000..92dd0543 --- /dev/null +++ b/src/keyring/keyring_kmip_ereport.c @@ -0,0 +1,17 @@ + +#include "postgres.h" + +#include "keyring/keyring_kmip.h" +#include "catalog/keyring_min.h" + +void kmip_ereport(bool throw_error, const char *msg, int errCode) +{ + if (errCode != 0) + { + ereport(throw_error, msg, errCode); + } + else + { + ereport(throw_error, msg); + } +} \ No newline at end of file diff --git a/src/libkmip b/src/libkmip new file mode 160000 index 00000000..f3f21ceb --- /dev/null +++ b/src/libkmip @@ -0,0 +1 @@ +Subproject commit f3f21ceb32bef8ce8fb36e25c6ae4831f7689e02 diff --git a/src/pg_tde.c b/src/pg_tde.c index 1d993ae2..223dc7cb 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -31,6 +31,7 @@ #include "catalog/tde_principal_key.h" #include "keyring/keyring_file.h" #include "keyring/keyring_vault.h" +#include "keyring/keyring_kmip.h" #include "utils/builtins.h" #include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" @@ -125,6 +126,7 @@ _PG_init(void) SetupTdeDDLHooks(); InstallFileKeyring(); InstallVaultV2Keyring(); + InstallKmipKeyring(); RegisterCustomRmgr(RM_TDERMGR_ID, &tdeheap_rmgr); RegisterStorageMgr();