Update Trino to 358

nineinchnick · Jun 16, 2021 · 5e53296 · 5e53296
1 parent 5499d49
commit 5e53296
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 24 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM trinodb/trino:357
+FROM trinodb/trino:358
 
 ARG VERSION
 

diff --git a/pom.xml b/pom.xml
@@ -57,7 +57,7 @@
         <air.check.fail-checkstyle>true</air.check.fail-checkstyle>
         <air.check.skip-checkstyle>false</air.check.skip-checkstyle>
 
-        <dep.trino.version>357</dep.trino.version>
+        <dep.trino.version>358</dep.trino.version>
         <dep.airlift.version>206</dep.airlift.version>
         <dep.packaging.version>${dep.airlift.version}</dep.packaging.version>
         <dep.errorprone.version>2.7.1</dep.errorprone.version>

diff --git a/trino-rest-github/README.md b/trino-rest-github/README.md
@@ -41,12 +41,12 @@ docker run \
   -p 8080:8080 \
   --name trino \
   -d \
-  trinodb/trino:357
+  trinodb/trino:358
 ```
 
 Connect to that server using:
 ```bash
-docker run -it --rm --link trino trinodb/trino:357 trino --server trino:8080 --catalog github --schema default
+docker run -it --rm --link trino trinodb/trino:358 trino --server trino:8080 --catalog github --schema default
 ```
 
 # Authentication and rate limits

diff --git a/trino-rest-github/src/main/java/pl/net/was/rest/github/GithubRest.java b/trino-rest-github/src/main/java/pl/net/was/rest/github/GithubRest.java
@@ -1423,6 +1423,7 @@ public Optional<LimitApplicationResult<ConnectorTableHandle>> applyLimit(
                         restTable.getConstraint(),
                         (int) Math.min(limit, Integer.MAX_VALUE),
                         restTable.getSortOrder().isPresent() ? restTable.getSortOrder().get() : null),
+                true,
                 true));
     }
 
@@ -1465,7 +1466,7 @@ public Optional<TopNApplicationResult<ConnectorTableHandle>> applyTopN(
                 limit,
                 sortItems);
 
-        return Optional.of(new TopNApplicationResult<>(sortedTableHandle, true));
+        return Optional.of(new TopNApplicationResult<>(sortedTableHandle, true, true));
     }
 
     @Override

diff --git a/trino-rest-github/src/main/java/pl/net/was/rest/github/Sync.java b/trino-rest-github/src/main/java/pl/net/was/rest/github/Sync.java
@@ -632,15 +632,19 @@ private static void syncArtifacts(Options options)
             // CREATE INDEX ON artifacts(owner, repo);
             // CREATE INDEX ON artifacts(run_id);
 
+            // get largest run id of those with an artifact and move up
             // TODO because dynamic filtering is not supported yet, CROSS JOIN LATERAL between runs and artifacts would not push down filter on run_id
             String runsQuery = "SELECT r.id " +
                     "FROM " + destSchema + ".runs r " +
                     "LEFT JOIN " + destSchema + ".artifacts a ON a.run_id = r.id " +
-                    "WHERE r.status = 'completed' AND r.created_at > NOW() - INTERVAL '2' MONTH AND r.id < ? " +
+                    "WHERE r.status = 'completed' AND r.created_at > NOW() - INTERVAL '2' MONTH " +
                     "GROUP BY r.id " +
-                    "HAVING COUNT(a.id) = 0 " +
+                    "HAVING COUNT(a.id) != 0 " +
                     "ORDER BY r.id DESC LIMIT 1";
-            PreparedStatement idStatement = conn.prepareStatement("SELECT min(r.id) FROM (" + runsQuery + ") r");
+            PreparedStatement idStatement = conn.prepareStatement("SELECT r.id " +
+                    "FROM " + destSchema + ".runs r " +
+                    "WHERE r.id > (" + runsQuery + ") " +
+                    "ORDER BY r.id ASC");
 
             String query = "INSERT INTO " + destSchema + ".artifacts " +
                     "SELECT src.* " +
@@ -651,22 +655,17 @@ private static void syncArtifacts(Options options)
             insertStatement.setString(2, options.owner);
             insertStatement.setString(3, options.repo);
 
-            long previousId = Long.MAX_VALUE;
-            while (true) {
-                idStatement.setLong(1, previousId);
-                log.info("Checking for next runs with jobs without artifacts");
-                ResultSet resultSet = idStatement.executeQuery();
-                if (!resultSet.next()) {
-                    break;
-                }
-                previousId = resultSet.getLong(1);
-                if (resultSet.wasNull()) {
-                    break;
-                }
+            log.info("Fetching run ids to get artifacts for");
+            if (!idStatement.execute()) {
+                return;
+            }
+            ResultSet resultSet = idStatement.getResultSet();
+            while (resultSet.next()) {
+                long runId = resultSet.getLong(1);
+                insertStatement.setLong(1, runId);
+                insertStatement.setLong(4, runId);
 
-                insertStatement.setLong(1, previousId);
-                insertStatement.setLong(4, previousId);
-                log.info("Fetching artifacts");
+                log.info(format("Fetching artifacts for jobs of run %d", runId));
                 long startTime = System.currentTimeMillis();
                 int rows = retryExecute(insertStatement);
                 log.info(format("Inserted %d rows, took %s", rows, Duration.ofMillis(System.currentTimeMillis() - startTime)));

diff --git a/trino-rest-github/src/main/java/pl/net/was/rest/github/filter/FilterApplier.java b/trino-rest-github/src/main/java/pl/net/was/rest/github/filter/FilterApplier.java
@@ -106,7 +106,8 @@ else if (!currentConstraint.getDomains().get().containsKey(column)) {
                         currentConstraint,
                         table.getLimit(),
                         table.getSortOrder().isPresent() ? table.getSortOrder().get() : null),
-                constraint));
+                constraint,
+                true));
     }
 
     private TupleDomain<ColumnHandle> normalizeConstraint(RestColumnHandle column, FilterType supportedFilter, TupleDomain<ColumnHandle> constraint)