Fixing errors with OpenSSL 1.0/1.1

Issue: there's a slight behavior difference between OpenSSL 1.1 and 3.0
in regards to padding, and how the EVP encryption/decryption functions
exactly behave.

Our code also incorrectly used these functions until now, which by chance
worked correctly with 3.0, but caused decryption issues with 1.0/1.1.

This commit:

* Fixes the issue
* Adds a few related assertions
* Adds an Ubuntu 20.04 (OpenSSL 1.1) github runner to verify that everything
  works correctly with 1.1.

.github/workflows/postgresql-16-src-make-ssl11.yml

@@ -0,0 +1,92 @@
+name: postgresql-16-src-make-ssl11
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  build:
+    name: pg-16-src-make-test-ssl11
+    runs-on: ubuntu-20.04
+    steps:
+
+
+      - name: Remove old postgres
+        run: |
+          sudo apt purge postgresql-client-common postgresql-common \
+            postgresql postgresql*
+          sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
+           /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
+           /etc/postgresql
+          sudo rm -f /usr/bin/pg_config
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
+            zlib1g-dev libssl-dev libpam0g-dev bison flex \
+            libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
+            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
+            libsystemd-dev gettext tcl-dev libperl-dev pkg-config \
+            libselinux1-dev python3-dev \
+            uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+
+      - name: Clone postgres repository
+        uses: actions/checkout@v2
+        with:
+          repository: 'postgres/postgres'
+          ref: 'a81e5516fa4bc53e332cb35eefe231147c0e1749'
+          path: 'src'
+
+      - name: Clone postgres-tde-ext repository
+        uses: actions/checkout@v2
+        with:
+          path: 'src/contrib/postgres-tde-ext'
+
+      - name: Create pgsql dir
+        run: mkdir -p /opt/pgsql
+
+      - name: Build postgres
+        run: |
+          ./configure --with-openssl  --enable-tap-tests=no
+          make -j
+          sudo make install
+        working-directory: src
+
+      - name: Build postgres-tde-ext
+        run: |
+          ./configure
+          make -j
+          sudo make install
+        working-directory: src/contrib/postgres-tde-ext
+
+      - name: Start postgresql cluster with pg_tde
+        run: |
+          export PATH="/usr/local/pgsql/bin:$PATH"
+          sudo cp /usr/local/pgsql/bin/pg_config /usr/bin
+          initdb -D /opt/pgsql/data
+          echo "shared_preload_libraries = 'pg_tde'" >> \
+            /opt/pgsql/data/postgresql.conf
+          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
+            /opt/pgsql/data/postgresql.conf
+          cp src/contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
+          pg_ctl -D /opt/pgsql/data -l logfile start
+
+      - name: Test postgres-tde-ext
+        run: |
+          make installcheck
+        working-directory: src/contrib/postgres-tde-ext
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/contrib/postgres-tde-ext/regression.diffs
+            logfile
+          retention-days: 3
+
+      - name: Report on test fail 2
+        if: ${{ failure() }}
+        run: |
+          cat src/contrib/postgres-tde-ext/regression.diffs

src/encryption/enc_aes.c

@@ -102,6 +102,7 @@ AesRun2(EVP_CIPHER_CTX** ctxPtr, int enc, const unsigned char* key, const unsign
 
 static void AesRun(int enc, const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len)
 {
+	int out_len2 = 0;
 	EVP_CIPHER_CTX* ctx = NULL;
 	ctx = EVP_CIPHER_CTX_new();
 	EVP_CIPHER_CTX_init(ctx);
@@ -119,6 +120,9 @@ static void AesRun(int enc, const unsigned char* key, const unsigned char* iv, c
 		goto cleanup;
 	}
 
+	EVP_CIPHER_CTX_set_padding(ctx, 0);
+	Assert(in_len % cipher_block_size == 0);
+
 	if(EVP_CipherUpdate(ctx, out, out_len, in, in_len) == 0)
 	{
 		#ifdef FRONTEND
@@ -130,7 +134,7 @@ static void AesRun(int enc, const unsigned char* key, const unsigned char* iv, c
 		goto cleanup;
 	}
 
-	if(EVP_CipherFinal_ex(ctx, out, out_len) == 0)
+	if(EVP_CipherFinal_ex(ctx, out + *out_len, &out_len2) == 0)
 	{
 		#ifdef FRONTEND
 			fprintf(stderr, "ERROR: EVP_CipherFinal_ex failed. OpenSSL error: %s\n", ERR_error_string(ERR_get_error(), NULL));
@@ -141,6 +145,9 @@ static void AesRun(int enc, const unsigned char* key, const unsigned char* iv, c
 		goto cleanup;
 	}
 
+	*out_len += out_len2;
+	Assert(in_len == *out_len);
+
 cleanup:
  	EVP_CIPHER_CTX_cleanup(ctx);
  	EVP_CIPHER_CTX_free(ctx);