feat(#9691): plug nouveau APIs with API lifecycle

api/src/db.js

@@ -149,6 +149,17 @@ if (UNIT_TEST_ENV) {
       });
   };
 
+  // TODO: couch is complaining about `Content-Type must be application/json`
+  module.exports.compactView = (db, ddoc) => request.post({
+    url: `${environment.serverUrl}/${db}/_compact/${ddoc}`,
+    json: true,
+  });
+
+  module.exports.compactNouveau = () => request.post({
+    url: `${environment.couchUrl}/_nouveau_cleanup`,
+    json: true,
+  });
+
   const saveDocs = async (db, docs) => {
     try {
       return await db.bulkDocs(docs);

api/src/services/setup/check-install.js

@@ -5,7 +5,7 @@ const ddocsService = require('./ddocs');
 const { DATABASES } = require('./databases');
 const startupLog = require('./startup-log');
 
-
+// TODO
 const checkInstallForDb = async (database) => {
   const check = {};
   const allDdocs = await ddocsService.getDdocs(database);
@@ -14,8 +14,8 @@ const checkInstallForDb = async (database) => {
   const liveDdocs = allDdocs.filter(ddoc => !ddocsService.isStaged(ddoc._id));
   const stagedDdocs = allDdocs.filter(ddoc => ddocsService.isStaged(ddoc._id));
 
-
   const liveDdocsCheck = ddocsService.compareDdocs(bundledDdocs, liveDdocs);
+  console.log("liveDdocsCheck.different", liveDdocsCheck.different);
   check.missing = liveDdocsCheck.missing;
   check.different = liveDdocsCheck.different;
 
@@ -69,7 +69,7 @@ const checkInstall = async () => {
     ddocValidation.push(await checkInstallForDb(database));
   }
 
-  const allDbsUpToDate = ddocValidation.every(check => check.upToDate);
+  const allDbsUpToDate = ddocValidation.every(check => console.log({check}) || check.upToDate);
   if (allDbsUpToDate) {
     logger.info('Installation valid.');
     await upgradeUtils.interruptPreviousUpgrade();

api/src/services/setup/databases.js

@@ -5,12 +5,21 @@ const db = require('../../db');
  * @property {string} name
  * @property {PouchDB} db
  * @property {string} jsonFileName
+ * @property {string[]?} ddocs
  */
 
 const MEDIC_DATABASE = {
   name: environment.db,
   db: db.medic,
   jsonFileName: 'medic.json',
+  ddocs: [
+    'medic',
+    'medic-admin',
+    'medic-client',
+    'medic-conflicts',
+    'medic-scripts',
+    'medic-sms',
+  ],
 };
 
 /**
@@ -22,6 +31,7 @@ const DATABASES = [
     name: `${environment.db}-sentinel`,
     db: db.sentinel,
     jsonFileName: 'sentinel.json',
+    ddocs: ['sentinel'],
   },
   {
     name: `${environment.db}-logs`,
@@ -32,11 +42,13 @@ const DATABASES = [
     name: `${environment.db}-users-meta`,
     db: db.medicUsersMeta,
     jsonFileName: 'users-meta.json',
+    ddocs: ['users-meta'],
   },
   {
     name: `_users`,
     db: db.users,
     jsonFileName: 'users.json',
+    ddocs: ['users'],
   },
 ];
 

api/src/services/setup/upgrade-steps.js

@@ -102,7 +102,8 @@ const indexStagedViews = async () => {
   const viewsToIndex = await viewIndexer.getViewsToIndex();
   const viewIndexingPromise = viewIndexer.indexViews(viewsToIndex);
   const stopQueryingIndexers = viewIndexerProgress.log();
-  await viewIndexingPromise;
+  const nouveauIndexingPromise = Promise.resolve();
+  await Promise.all([viewIndexingPromise, nouveauIndexingPromise]);
   stopQueryingIndexers();
 };
 

api/src/services/setup/utils.js

@@ -55,15 +55,20 @@ const deleteStagedDdocs = async () => {
 const cleanup = () => {
   for (const database of DATABASES) {
     logger.info(`Running DB compact and view cleanup for ${database.name}`);
+    const ddocs = database.ddocs ?? [];
     Promise
       .all([
         database.db.compact(),
         database.db.viewCleanup(),
+        ...ddocs.map((ddoc) => db.compactView(database.name, ddoc)),
       ])
       .catch(err => {
         logger.error('Error while running cleanup: %o', err);
       });
   }
+  db.compactNouveau().catch(err => {
+    logger.error('Error while running cleanup: %o', err);
+  });
 };
 /**
  * Updates json ddocs to add the :staged: ddoc name prefix

api/src/services/setup/view-indexer-progress.js

@@ -56,6 +56,7 @@ const updateRunningTasks = (indexers, activeTasks = []) => {
       indexers.push(indexer);
     }
 
+    console.log(`${task.node}-${task.pid} progress`, task.progress);
     indexer.tasks[`${task.node}-${task.pid}`] = task.progress;
   });
 };
@@ -89,11 +90,23 @@ const logIndexersProgress = (indexers) => {
 const getIndexers = async (indexers = []) => {
   try {
     const activeTasks = await db.activeTasks();
-    const tasks = activeTasks.filter(task => task.type === 'indexer' && DDOC_PREFIX.test(String(task.design_document)));
+    const tasks = activeTasks.filter(task => {
+      const isFirstNouveauIndexingTask = task.type === 'search_indexer' &&
+        task.design_document === '_design/medic-nouveau';
+      if (isFirstNouveauIndexingTask) {
+        return true;
+      }
+
+      return DDOC_PREFIX.test(String(task.design_document));
+    });
     // We assume all previous tasks have finished.
+    console.log("indexers 1", indexers);
     indexers.forEach(setTasksToComplete);
+    console.log("indexers 2", indexers);
     updateRunningTasks(indexers, tasks);
+    console.log("indexers 3", indexers);
     indexers.forEach(calculateAverageProgress);
+    console.log("indexers 4", indexers);
     return indexers;
   } catch (err) {
     logger.error('Error while querying active tasks: %o', err);

api/src/services/setup/view-indexer.js

@@ -12,13 +12,17 @@ let continueIndexing;
 const indexViews = async (viewsToIndex) => {
   continueIndexing = true;
 
+  console.log("viewsToIndex", viewsToIndex);
   if (!Array.isArray(viewsToIndex)) {
+    console.log("not array");
     await upgradeLogService.setIndexed();
     return;
   }
 
   await upgradeLogService.setIndexing();
   const indexResult = await Promise.all(viewsToIndex.map(indexView => indexView()));
+  console.log("indexResult", indexResult);
+  console.log("continueIndexing", continueIndexing);
   if (continueIndexing) {
     await upgradeLogService.setIndexed();
   }
@@ -38,14 +42,19 @@ const getViewsToIndex = async () => {
   for (const database of DATABASES) {
     const stagedDdocs = await ddocsService.getStagedDdocs(database);
     stagedDdocs.forEach(ddoc => {
-      if (!ddoc.views || !_.isObject(ddoc.views)) {
-        return;
+      if (ddoc.views && !_.isObject(ddoc.views)) {
+        const ddocViewIndexPromises = Object
+          .keys(ddoc.views)
+          .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName));
+        viewsToIndex.push(...ddocViewIndexPromises);
       }
 
-      const ddocViewIndexPromises = Object
-        .keys(ddoc.views)
-        .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName));
-      viewsToIndex.push(...ddocViewIndexPromises);
+      if (ddoc.nouveau && !_.isObject(ddoc.nouveau)) {
+        const ddocNouveauIndexPromises = Object
+          .keys(ddoc.nouveau)
+          .map(indexName => indexNouveauIndex.bind({}, database.name, ddoc._id, indexName));
+        viewsToIndex.push(...ddocNouveauIndexPromises);
+      }
     });
   }
   return viewsToIndex;
@@ -79,7 +88,36 @@ const indexView = async (dbName, ddocId, viewName) => {
   } while (continueIndexing);
 };
 
+/**
+ * Returns a promise that resolves when a nouveau index is indexed.
+ * Retries querying the index until no error is thrown
+ * @param {String} dbName
+ * @param {String} ddocId
+ * @param {String} indexName
+ * @return {Promise}
+ */
+const indexNouveauIndex = async (dbName, ddocId, indexName) => {
+  do {
+    try {
+      return await request.get({
+        uri: `${environment.serverUrl}/${dbName}/_design/${ddocId}/_nouveau/${indexName}`,
+        json: true,
+        qs: { q: '*:*', limit: 1 },
+      });
+    } catch (requestError) {
+      if (!continueIndexing) {
+        return;
+      }
+
+      if (!requestError || !requestError.error || !SOCKET_TIMEOUT_ERROR_CODE.includes(requestError.error.code)) {
+        throw requestError;
+      }
+    }
+  } while (continueIndexing);
+};
+
 const stopIndexing = () => {
+  console.trace("************ stopIndexing");
   continueIndexing = false;
 };
 

ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js

@@ -1,4 +1,5 @@
 function (doc) {
+  'different 2';
   var skip = ['_id', '_rev', 'type', 'refid', 'geolocation'];
   var toIndex = '';