Added support for nightly versions

client/factory.go

@@ -7,9 +7,10 @@ import (
 
 func New() Client {
 	return &ApiClient{
-		Host: "https://raw.githubusercontent.com/NiclasHaderer/duckdb-version-manager/main/",
+		// Host: "https://raw.githubusercontent.com/NiclasHaderer/duckdb-version-manager/main/",
+		Host: "http://127.0.0.1:8000",
 		Client: &http.Client{
-			Timeout: 5 * time.Second,
+			Timeout: 5 * time.Minute,
 		},
 		BasePath: "/versions",
 	}

client/utils.go

@@ -15,6 +15,13 @@ func toVersionList(dict models.VersionDict) models.VersionList {
 
 	// Sort the version list
 	sort.Slice(versionList, func(i, j int) bool {
+		if versionList[i].Version == "nightly" {
+			return true
+		}
+		if versionList[j].Version == "nightly" {
+			return false
+
+		}
 		v1, _ := version.NewVersion(versionList[i].Version)
 		v2, _ := version.NewVersion(versionList[j].Version)
 

compile.sh

@@ -7,6 +7,7 @@ GOOS_ARRAY=("darwin" "darwin" "linux" "linux" "windows" "windows")
 GOARCH_ARRAY=("amd64" "arm64" "amd64" "arm" "amd64" "arm")
 
 # Create output directory
+rm -r $OUTPUT_DIR
 mkdir -p $OUTPUT_DIR
 
 # Loop through architectures

pyproject.toml

@@ -9,6 +9,7 @@ readme = "README.md"
 python = "^3.11"
 pygithub = "^2.2.0"
 black = "^24.2.0"
+httpx = "^0.27.0"
 
 
 [build-system]

scripts/crawl_releases.py → scripts/extract_releases.py

@@ -6,6 +6,7 @@
 from typing import Literal
 
 import github.Auth
+import httpx
 from github import Github
 from github.GitRelease import GitRelease
 from github.PaginatedList import PaginatedList
@@ -18,6 +19,7 @@
     ArchitectureType,
     PlatformType,
 )
+from scripts.nightly import NIGHTLY_RELEASE
 
 VERSIONS_BASE_FOLDER = Path(__file__).parent.parent / "versions"
 VERSIONS_INDIVIDUAL_FOLDER = VERSIONS_BASE_FOLDER / "tags"
@@ -47,21 +49,21 @@ def get_asset_type_from_name(asset_name: str, mode: Literal["duck-vm", "duck-db"
 
     architecture: ArchitectureType
     if "amd64" in asset_name:
-        architecture = ArchitectureType.ArchitectureX86
+        architecture = "ArchitectureX86"
     elif "aarch64" in asset_name or "arm64" in asset_name:
-        architecture = ArchitectureType.ArchitectureArm64
+        architecture = "ArchitectureArm64"
     elif "universal" in asset_name:
-        architecture = ArchitectureType.ArchitectureUniversal
+        architecture = "ArchitectureUniversal"
     else:
         return None
 
     platform: PlatformType
     if "windows" in asset_name:
-        platform = PlatformType.PlatformWindows
+        platform = "PlatformWindows"
     elif "osx" in asset_name or "darwin" in asset_name:
-        platform = PlatformType.PlatformMac
+        platform = "PlatformMac"
     elif "linux" in asset_name:
-        platform = PlatformType.PlatformLinux
+        platform = "PlatformLinux"
     else:
         return None
 
@@ -86,7 +88,7 @@ def save_duckdb_releases():
             asset_info = get_asset_type_from_name(asset.name, "duck-db")
             if asset_info is None:
                 continue
-            serializable_release["platforms"][asset_info["platform"].name][asset_info["architecture"].name] = {
+            serializable_release["platforms"][asset_info["platform"]][asset_info["architecture"]] = {
                 "downloadUrl": asset.browser_download_url
             }
 
@@ -96,10 +98,22 @@ def save_duckdb_releases():
         with open(VERSIONS_INDIVIDUAL_FOLDER / f"{tag_name}.json", "w") as f:
             f.write(json.dumps(serializable_release, indent=2))
 
+    with open(VERSIONS_INDIVIDUAL_FOLDER / f"nightly.json", "w") as f:
+        f.write(json.dumps(NIGHTLY_RELEASE, indent=2))
+
+    version_list["nightly"] = "tags/nightly.json"
+
     with open(VERSIONS_BASE_FOLDER / "versions.json", "w") as f:
         f.write(json.dumps(version_list, indent=2))
 
 
+def validate_nightly_urls(nightly: Release):
+    for platform, architectures in nightly["platforms"].items():
+        for architecture, info in architectures.items():
+            if httpx.get(info["downloadUrl"], follow_redirects=True).status_code != 200:
+                raise ValueError(f"URL {info['downloadUrl']} is not valid")
+
+
 def save_latest_duck_vm_release():
     repo_name = "niclashaderer/duckdb-version-manager"
     releases = get_all_releases_from(repo_name)
@@ -117,7 +131,7 @@ def save_latest_duck_vm_release():
         asset_info = get_asset_type_from_name(asset.name, "duck-vm")
         if asset_info is None:
             continue
-        serializable_release["platforms"][asset_info["platform"].name][asset_info["architecture"].name] = {
+        serializable_release["platforms"][asset_info["platform"]][asset_info["architecture"]] = {
             "downloadUrl": asset.browser_download_url
         }
 
@@ -126,6 +140,7 @@ def save_latest_duck_vm_release():
 
 
 def main():
+    validate_nightly_urls(NIGHTLY_RELEASE)
     save_latest_duck_vm_release()
     save_duckdb_releases()
 

scripts/models.py

@@ -1,5 +1,4 @@
-from enum import Enum, auto
-from typing import TypedDict
+from typing import TypedDict, Literal
 
 VersionList = dict[str, str]
 
@@ -8,27 +7,21 @@ class ArchitectureReleaseInformation(TypedDict):
     downloadUrl: str
 
 
-Architectures = dict[str, ArchitectureReleaseInformation]
+ArchitectureType = Literal["ArchitectureX86", "ArchitectureArm64", "ArchitectureUniversal"]
 
-Platforms = dict[str, Architectures]
 
+PlatformType = Literal["PlatformWindows", "PlatformMac", "PlatformLinux"]
 
-class Release(TypedDict):
-    version: str
-    name: str
-    platforms: Platforms
 
+Architectures = dict[ArchitectureType, ArchitectureReleaseInformation]
 
-class ArchitectureType(Enum):
-    ArchitectureX86 = auto()
-    ArchitectureArm64 = auto()
-    ArchitectureUniversal = auto()
+Platforms = dict[PlatformType, Architectures]
 
 
-class PlatformType(Enum):
-    PlatformWindows = auto()
-    PlatformMac = auto()
-    PlatformLinux = auto()
+class Release(TypedDict):
+    version: str
+    name: str
+    platforms: Platforms
 
 
 class AssetInformation(TypedDict):

scripts/nightly.py

@@ -0,0 +1,26 @@
+from scripts.models import Release
+
+NIGHTLY_RELEASE: Release = {
+    "version": "nightly",
+    "name": "nightly",
+    "platforms": {
+        "PlatformWindows": {
+            "ArchitectureX86": {
+                "downloadUrl": "https://artifacts.duckdb.org/latest/duckdb-binaries-windows.zip",
+            },
+        },
+        "PlatformLinux": {
+            "ArchitectureX86": {
+                "downloadUrl": "https://artifacts.duckdb.org/latest/duckdb-binaries-linux.zip",
+            },
+            "ArchitectureArm64": {
+                "downloadUrl": "https://artifacts.duckdb.org/latest/duckdb-binaries-linux-aarch64.zip",
+            },
+        },
+        "PlatformMac": {
+            "ArchitectureUniversal": {
+                "downloadUrl": "https://artifacts.duckdb.org/latest/duckdb-binaries-osx.zip",
+            },
+        },
+    },
+}

utils/download.go

@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"duckdb-version-manager/models"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -13,6 +14,16 @@ import (
 	"time"
 )
 
+func getZipFileContent(file *zip.File) ([]byte, error) {
+	fileReader, err := file.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer fileReader.Close()
+
+	return io.ReadAll(fileReader)
+}
+
 func saveZip(body []byte, dest string) error {
 	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
 	if err != nil {
@@ -22,28 +33,26 @@ func saveZip(body []byte, dest string) error {
 	for _, file := range zipReader.File {
 		fileName := file.Name
 		if fileName == "duckdb" {
-			fileReader, err := file.Open()
+			fileContent, err := getZipFileContent(file)
 			if err != nil {
 				return err
 			}
-			defer fileReader.Close()
 
-			fileContent, err := io.ReadAll(fileReader)
-			if err != nil {
-				return err
-			}
-
-			err = os.WriteFile(dest, fileContent, 0700)
+			return os.WriteFile(dest, fileContent, 0700)
+		}
+		if strings.Contains(fileName, "duckdb_cli") && strings.HasSuffix(fileName, ".zip") {
+			fileContent, err := getZipFileContent(file)
 			if err != nil {
 				return err
 			}
+			return saveZip(fileContent, dest)
 		}
 	}
-	return nil
+	return errors.New("no duckdb binary found in zip")
 }
 
 func DownloadUrlTo(url string, dest string, isZip bool) error {
-	httpClient := &http.Client{Timeout: 5 * time.Second}
+	httpClient := &http.Client{Timeout: 5 * time.Minute}
 	resp, err := httpClient.Get(url)
 	if err != nil {
 		return err