Merge pull request #323 from mff-uk/feature/wikidata-adapter

Wikidata experimental adapter

applications/client/package.json

@@ -15,6 +15,7 @@
     "@dataspecer/sgov-adapter": "*",
     "@dataspecer/shacl": "*",
     "@dataspecer/sparql-query": "*",
+    "@dataspecer/wikidata-experimental-adapter": "*",
     "@dataspecer/xml": "*",
     "@emotion/css": "^11.10.6",
     "@emotion/react": "^11.10.6",

applications/client/src/editor/configuration/adapters/get-adapter.ts

@@ -2,13 +2,25 @@ import {PrefixIriProvider} from "@dataspecer/core/cim";
 import {httpFetch} from "@dataspecer/core/io/fetch/fetch-browser";
 import {RdfsFileAdapter} from "@dataspecer/rdfs-adapter";
 import {SgovAdapter} from "@dataspecer/sgov-adapter";
+import {WikidataAdapter} from "@dataspecer/wikidata-experimental-adapter";
 
 // todo: this is a temporary solution
 export const getAdapter = (urls: string[]) => {
     const iriProvider = new PrefixIriProvider();
-    const cimAdapter = urls.length > 0 ?
-        new RdfsFileAdapter(urls, httpFetch) :
-        new SgovAdapter("https://slovník.gov.cz/sparql", httpFetch);
+
+    if (urls.length === 0) {
+        const cimAdapter = new SgovAdapter("https://slovník.gov.cz/sparql", httpFetch);
+        cimAdapter.setIriProvider(iriProvider);
+        return {iriProvider, cimAdapter};
+    }
+
+    if (urls.length === 1 && urls[0] === "https://dataspecer.com/adapters/wikidata") {
+        const cimAdapter = new WikidataAdapter(httpFetch);
+        cimAdapter.setIriProvider(iriProvider);
+        return {iriProvider, cimAdapter};
+    }
+
+    const cimAdapter = new RdfsFileAdapter(urls, httpFetch);
     cimAdapter.setIriProvider(iriProvider);
     return {iriProvider, cimAdapter};
 }

applications/client/src/manager/routes/specification/conceptual-model-sources.tsx

@@ -1,6 +1,6 @@
 import {
     Box, Button,
-    Card, CardContent, Collapse,
+    Card, CardContent, Chip, Collapse,
     FormControl,
     FormControlLabel,
     FormLabel,
@@ -39,16 +39,20 @@ export const ConceptualModelSources: FC<ConceptualModelSourcesProps> = (props) =
     }
 }
 
+const WIKIDATA_ADAPTER = "https://dataspecer.com/adapters/wikidata";
+
 const ConceptualModelSourcesWithData: FC<ConceptualModelSourcesWithDataProps> = ({dataSpecificationIri, dataSpecification}) => {
     const {dataSpecifications, setDataSpecifications} = useContext(DataSpecificationsContext);
     const backendConnector = useContext(BackendConnectorContext);
     const {enqueueSnackbar} = useSnackbar();
 
-    const [radio, setRadio] = useState((dataSpecification.cimAdapters?.length ?? 0) === 0 ? "sgov" : "files");
-    const [urls, setUrls] = useState(dataSpecification.cimAdapters ?? []);
+    const adapters = dataSpecification.cimAdapters ?? [];
+
+    const [radio, setRadio] = useState(adapters.length === 0 ? "sgov" : (adapters.length === 1 && adapters[0] === WIKIDATA_ADAPTER ? "wikidata" : "files"));
+    const [urls, setUrls] = useState(dataSpecification.cimAdapters);
 
     const handleSave = useCallback(async () => {
-        const cimAdapters = radio === "sgov" ? [] : urls.filter(url => url.length > 0);
+        const cimAdapters = radio === "sgov" ? [] : (radio === "wikidata" ? [WIKIDATA_ADAPTER] : urls.filter(url => url.length > 0));
         const newSpecification = await backendConnector.updateDataSpecification(dataSpecificationIri, {cimAdapters});
         setDataSpecifications({...dataSpecifications, [newSpecification.iri]: newSpecification});
         enqueueSnackbar("Sources configuration saved", {variant: "success"});
@@ -71,6 +75,7 @@ const ConceptualModelSourcesWithData: FC<ConceptualModelSourcesWithDataProps> =
                         >
                             <FormControlLabel value="sgov" control={<Radio />} label="slovník.gov.cz" />
                             <FormControlLabel value="files" control={<Radio />} label="RDF files" />
+                            <FormControlLabel value="wikidata" control={<Radio />} label={<>Wikidata <Chip label="experimental" variant="outlined" color="warning" size="small" /></>} />
                         </RadioGroup>
                     </FormControl>
 

packages/wikidata-experimental-adapter/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+lib
+!.gitignore

packages/wikidata-experimental-adapter/README.md

@@ -0,0 +1,29 @@
+# Wikidata experimental adapter
+
+## What can it do?
+
+- full hierarchy
+  - children
+  - parents
+- surroundings
+  - parents in height 1
+  - children in depth 1
+  - attributes (wikidata properties that do not point to items based on subject contraint)
+  - associations
+    - internaly there are outward pimAssociations but I need to add pimClasses for the endpoints.
+## How to start it up?
+
+1. `> git clone repository`
+2. `> cd repository`
+3. `> git fetch` just in case
+4. `> git switch feature/wikidata-adapter`
+5. `> npm install`
+6. `> npx lerna bootstrap`
+7. `> npx lerna run build`
+8. `> cd ./application/client`
+9. `> npm run build:watch`
+10. open your localhost at 3000
+11. to see changes in code do:
+   1.  from root run `> cd packages/wikidata-experimental-feature`
+   2.  `> npm run build:watch` simultaneously with 7. command inside client.
+   3.  to build queries run `> npm run prebuild` and then again `> npm run build:watch`

packages/wikidata-experimental-adapter/build/compile-sparql.js

@@ -0,0 +1,32 @@
+const { join, dirname } = require('path');
+const { existsSync, readdirSync, lstatSync, readFileSync, writeFileSync, mkdirSync } = require('fs');
+const { process } = require('./sparql-loader');
+
+const found = [];
+
+function fromDir(startPath, filter, found) {
+    if (!existsSync(startPath)) {
+        return;
+    }
+
+    var files = readdirSync(startPath);
+    for (var i = 0; i < files.length; i++) {
+        var filename = join(startPath, files[i]);
+        var stat = lstatSync(filename);
+        if (stat.isDirectory()) {
+            fromDir(filename, filter, found); //recurse
+        } else if (filename.endsWith(filter)) {
+            found.push(filename);
+        };
+    };
+};
+
+fromDir('./src', '.sparql', found);
+
+for (const file of found) {
+    const rawData = readFileSync(file);
+    const transformed = process(rawData.toString()).code;
+    const newFile = 'lib' + file.substring('src'.length) + '.js';
+    mkdirSync(dirname(newFile), {recursive: true});
+    writeFileSync(newFile, transformed);
+}

packages/wikidata-experimental-adapter/build/prebuild.js

@@ -0,0 +1,2 @@
+require("rimraf").sync("lib");
+require("./compile-sparql");

packages/wikidata-experimental-adapter/build/sparql-loader.js

@@ -0,0 +1,29 @@
+// Converts .sparql files into .js files that can be imported
+
+function processSparql(input) {
+    const minified = input.replace(/^\s*#.*$/gm, "").replace(/\s+/g, " ");
+    const parts = minified
+        .split(/%[A-Z_]+%/)
+        .map(part => JSON.stringify(part));
+    const variables = [...minified
+        .matchAll(/%([A-Z_]+)%/g)]
+        .map(([,variable]) =>
+            variable
+                .toLowerCase()
+                .replace(/_+(.)/g, (_, chr) => chr.toUpperCase())
+        ).map(key => `p.${key}`);
+
+    // zip the arrays
+    const zipped = parts.flatMap((part, i) => [part, variables[i] ?? ""]);
+
+    return zipped.filter(p => p.length > 0).join(" +\n\t");
+}
+
+module.exports = {
+    process: content => ({
+        code: `"use strict";\n` +
+            `Object.defineProperty(exports, "__esModule", { value: true });\n` +
+            `var a = p => ${processSparql(content)};\n` +
+            `exports.default = a;\n`
+    })
+};

packages/wikidata-experimental-adapter/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  "verbose": true,
+  "moduleFileExtensions": [
+    "js",
+    "ts",
+  ],
+  "transform": {
+    "^.+\\.[t|j]sx?$": "ts-jest",
+    "\\.sparql$": "./build/sparql-loader.js",
+  },
+  "roots": [
+    "src",
+  ],
+};