Merge pull request #281 from mhbkb/backing_storage-Storing

Supports layered wheel cache.

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java

@@ -25,4 +25,12 @@ public class EmptyWheelCache implements WheelCache {
     public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails) {
         return Optional.empty();
     }
+
+    @Override
+    public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
+        return Optional.empty();
+    }
+
+    @Override
+    public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { }
 }

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java

@@ -51,6 +51,14 @@ public Optional<File> findWheel(String library, String version, PythonDetails py
         return findWheel(library, version, pythonDetails.getVirtualEnvInterpreter());
     }
 
+    @Override
+    public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
+        return Optional.empty();
+    }
+
+    @Override
+    public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { }
+
     /**
      * Find's a wheel in the wheel cache.
      *

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.gradle.python.wheel;
+
+import com.linkedin.gradle.python.extension.PythonDetails;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+
+public class LayeredWheelCache implements WheelCache, Serializable {
+
+    private static final Logger logger = Logging.getLogger(LayeredWheelCache.class);
+
+    private final Map<WheelCacheLayer, File> layeredCacheMap;
+    private final PythonAbiContainer pythonAbiContainer;
+
+    public LayeredWheelCache(Map<WheelCacheLayer, File> layeredCacheMap, PythonAbiContainer pythonAbiContainer) {
+        this.layeredCacheMap = layeredCacheMap;
+        this.pythonAbiContainer = pythonAbiContainer;
+    }
+
+    /**
+     * Find a wheel from all wheel cache layers.
+     *
+     * @param library       name of the library
+     * @param version       version of the library
+     * @param pythonDetails details on the python to find a wheel for
+     * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()}
+     */
+    @Override
+    public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails) {
+        // TODO: Make sure layeredCacheMap is a LinkedHashMap when we initialize it in the plugin.
+        for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) {
+            Optional<File> layerResult = findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer);
+
+            if (layerResult.isPresent()) {
+                return layerResult;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    /**
+     * Find wheel based on cache layer.
+     *
+     * @param library         name of the library
+     * @param version         version of the library
+     * @param pythonDetails   details on the python to find a wheel for
+     * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel
+     * @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()}
+     */
+    @Override
+    public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
+        return findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer);
+    }
+
+    /**
+     * Find a wheel from target layer.
+     *
+     * @param library          name of the library
+     * @param version          version of the library
+     * @param pythonExecutable python executable
+     * @param wheelCacheLayer  which {@link WheelCacheLayer} to fetch wheel
+     * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()}
+     */
+    public Optional<File> findWheelInLayer(String library, String version, File pythonExecutable, WheelCacheLayer wheelCacheLayer) {
+        File cacheDir = layeredCacheMap.get(wheelCacheLayer);
+
+        if (cacheDir == null) {
+            return Optional.empty();
+        }
+
+        /*
+         * NOTE: Careful here! The prefix *must* end with a hyphen.
+         * Otherwise 0.0.2 version will match 0.0.20.
+         * Both name and version of the package must replace hyphen with underscore.
+         * See PEP 427: https://www.python.org/dev/peps/pep-0427/
+         */
+        String wheelPrefix = (
+                library.replace("-", "_")
+                + "-"
+                + version.replace("-", "_")
+                + "-"
+        );
+        logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix);
+        File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl"));
+
+        if (files == null) {
+            return Optional.empty();
+        }
+
+        List<PythonWheelDetails> wheelDetails = Arrays.stream(files).map(PythonWheelDetails::fromFile)
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .collect(Collectors.toList());
+
+        logger.info("Wheels for version of library: {}", wheelDetails);
+
+        Optional<PythonWheelDetails> foundWheel = wheelDetails.stream()
+            .filter(it -> wheelMatches(pythonExecutable, it))
+            .findFirst();
+
+        logger.info("Found artifacts: {}", foundWheel);
+
+        return foundWheel.map(it -> it.getFile());
+    }
+
+    /**
+     * Store given wheel file to target layer.
+     *
+     * @param wheelFile       the wheel file to store
+     * @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel
+     */
+    @Override
+    public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) {
+        File cacheDir = layeredCacheMap.get(wheelCacheLayer);
+
+        if (wheelFile != null && cacheDir != null) {
+            try {
+                Files.copy(wheelFile.toPath(), new File(cacheDir, wheelFile.getName()).toPath());
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+    }
+
+    private boolean wheelMatches(File pythonExecutable, PythonWheelDetails wheelDetails) {
+        return pythonAbiContainer.matchesSupportedVersion(
+            pythonExecutable,
+            wheelDetails.getPythonTag(),
+            wheelDetails.getAbiTag(),
+            wheelDetails.getPlatformTag());
+    }
+}

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java

@@ -23,4 +23,23 @@
 
 public interface WheelCache extends Serializable {
     Optional<File> findWheel(String library, String version, PythonDetails pythonDetails);
+
+    /**
+     * Find wheel based on cache layer.
+     *
+     * @param library         name of the library
+     * @param version         version of the library
+     * @param pythonDetails   details on the python to find a wheel for
+     * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel
+     * @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()}
+     */
+    Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer);
+
+    /**
+     * Store given wheel file to target layer.
+     *
+     * @param wheelFile       the wheel file to store
+     * @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel
+     */
+    void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer);
 }

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCacheLayer.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.gradle.python.wheel;
+
+public enum WheelCacheLayer {
+    PROJECT_LAYER("projectLayer"),
+    HOST_LAYER("hostLayer");
+
+    private final String value;
+
+    WheelCacheLayer(String val) {
+        this.value = val;
+    }
+
+    @Override
+    public String toString() {
+        return value;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+}

pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.gradle.python.wheel
+
+
+import com.linkedin.gradle.python.extension.internal.DefaultPythonDetails
+import com.linkedin.gradle.python.wheel.internal.DefaultPythonAbiContainer
+import org.gradle.testfixtures.ProjectBuilder
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
+import spock.lang.Specification
+
+class LayeredWheelCacheTest extends Specification {
+
+    @Rule
+    TemporaryFolder temporaryFolder
+
+    private File projectLayerCache
+    private File hostLayerCache
+    private File pythonExec
+    private Map<WheelCacheLayer, File> cacheMap
+    private DefaultPythonDetails pythonDetails
+    private LayeredWheelCache cache
+
+    void setup() {
+        projectLayerCache = temporaryFolder.newFolder('project-cache')
+        hostLayerCache = temporaryFolder.newFolder('host-cache')
+
+        def virtualEnv = temporaryFolder.newFile('venv')
+        pythonExec = new File(virtualEnv, 'bin/python')
+
+        def formats = new DefaultPythonAbiContainer()
+        formats.addSupportedAbi(new AbiDetails(pythonExec, 'py2', 'none', 'any'))
+
+        pythonDetails = new DefaultPythonDetails(new ProjectBuilder().build(), virtualEnv)
+        cacheMap = [(WheelCacheLayer.PROJECT_LAYER): projectLayerCache, (WheelCacheLayer.HOST_LAYER): hostLayerCache]
+        cache = new LayeredWheelCache(cacheMap, formats)
+    }
+
+    def "can find Sphinx-1.6.3 in host layer"() {
+        setup:
+        new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile()
+
+        expect:
+        cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent()
+        !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent()
+    }
+
+    def "can find Sphinx-1.6.3 in project layer"() {
+        setup:
+        new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile()
+
+        expect:
+        cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent()
+        !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent()
+    }
+
+    def "can find Sphinx-1.6.3 in all layers"() {
+        setup:
+        new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile()
+        new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile()
+
+        expect:
+        cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent()
+        cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent()
+    }
+
+    def "can find Sphinx-1.6.3 despite layers"() {
+        setup:
+        new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile()
+
+        expect:
+        cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent()
+    }
+
+    def "cannot find Sphinx-1.6.3 if not put to any cache layer"() {
+        expect:
+        !cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent()
+    }
+
+    def "can find Sphinx-1.6.3 from target folder"() {
+        setup:
+        new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile()
+
+        expect:
+        cache.findWheelInLayer('Sphinx', '1.6.3', pythonExec, WheelCacheLayer.HOST_LAYER).isPresent()
+        !cache.findWheelInLayer('Sphinx', '1.6.3', pythonExec, WheelCacheLayer.PROJECT_LAYER).isPresent()
+    }
+
+    def "can store Sphinx-1.6.3 to target layer"() {
+        setup:
+        def wheelFile  = new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl')
+        wheelFile.createNewFile()
+        cache.storeWheel(wheelFile, WheelCacheLayer.HOST_LAYER)
+
+        expect:
+        cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent()
+    }
+}