diff --git a/README.md b/README.md index c84a24f..9ac2fe5 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,6 @@ Obviously, this depends on Kaggle services. When you're extending the API and mo or adding to those services, you should be working in your Kaggle mid-tier development environment. You'll run Kaggle locally, in the container, and test the Python code by running it in the container so it can connect to your local testing environment. -However, do not try to create a release from within the container. The code formatter -(`yapf3`) changes much more than intended. Also, run the following command to get `autogen.sh` installed: ```bash diff --git a/kaggle/api/kaggle_api_extended.py b/kaggle/api/kaggle_api_extended.py index 95cdeb8..66a9934 100644 --- a/kaggle/api/kaggle_api_extended.py +++ b/kaggle/api/kaggle_api_extended.py @@ -1,19 +1,19 @@ -#!/usr/bin/python -# -# Copyright 2024 Kaggle Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#!/usr/bin/python +# +# Copyright 2024 Kaggle Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/python # # Copyright 2024 Kaggle Inc @@ -305,6 +305,8 @@ class KaggleApi: already_printed_version_warning = False args = {} # DEBUG Add --local to use localhost + if os.environ.get('KAGGLE_API_ENVIRONMENT') == 'LOCALHOST': + args = {'--local'} # Kernels valid types valid_push_kernel_types = ['script', 'notebook'] diff --git a/pyproject.toml b/pyproject.toml index c244e2f..ae72a7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,11 +61,11 @@ install-autogen = """curl -fsSL --output /tmp/autogen.zip "https://github.com/mb # TODO: install in Mac/Windows install-yapf = """pip3 install yapf==0.40.2 --break-system-packages || echo 'yapf could not be installed'""" install-toml = """sudo apt-get install -y python3-toml || echo 'toml could not be installed'""" -install-java = """sudo apt-get install -y default-jre || echo 'java could not be installed'""" -install-deps = "hatch run install-unzip && hatch run install-autogen && hatch run install-yapf && hatch run install-toml && hatch run install-java" +install-deps = "hatch run install-unzip && hatch run install-autogen && hatch run install-yapf && hatch run install-toml" integration-test = "pytest {args:integration_tests}" compile = "./tools/GeneratePythonLibrary.sh" install = "./tools/GeneratePythonLibrary.sh --install" watch = "./tools/GeneratePythonLibrary.sh --watch" +test = "./tools/GeneratePythonLibrary.sh --test local" diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index c9a8136..1256729 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -289,6 +289,8 @@ class KaggleApi: already_printed_version_warning = False args = {} # DEBUG Add --local to use localhost + if os.environ.get('KAGGLE_API_ENVIRONMENT') == 'LOCALHOST': + args = {'--local'} # Kernels valid types valid_push_kernel_types = ['script', 'notebook'] diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 6c87905..96f0af6 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -251,7 +251,6 @@ def test_kernels_c_push(self): self.fail(f"kernels_push failed: {e}") def test_kernels_d_status(self): - # AssertionError: kernels_status failed: 404 Client Error: Not Found for url: http://localhost/api/v1/kernels/status?username=stevemessick&kernelslug=testing if self.kernel_slug == '': self.test_kernels_c_push() try: @@ -283,7 +282,6 @@ def test_kernels_e_list_files(self): self.fail(f"kernels_list_files failed: {e}") def test_kernels_f_output(self): - # AssertionError: kernels_output failed: 404 Client Error: Not Found for url: http://localhost/api/v1/kernels/output?username=stevemessick&kernelslug=testing fs = [] if self.kernel_slug == '': self.test_kernels_c_push() diff --git a/tools/GeneratePythonLibrary.sh b/tools/GeneratePythonLibrary.sh index 6942a7f..c6af85d 100755 --- a/tools/GeneratePythonLibrary.sh +++ b/tools/GeneratePythonLibrary.sh @@ -12,7 +12,7 @@ function usage { echo " --editable (-e): Make the installed package always reference your latest" echo " source code. Implies \"-i|--install\". Be aware that changes to the \"src\"" echo " directory won't be reflected. See the README for details." - echo " --test (-t) [$LOCAL_ENV|$PROD_ENV]: Run tests (python_api_tests.py) against http://localhost" + echo " --test (-t) [$LOCAL_ENV|$PROD_ENV]: Run tests (unit_tests.py) against http://localhost" echo " or https://www.kaggle.com." echo " --watch (-w): Run the script in watch mode. It will watch the files under the \"template\"" echo " directory and KaggleSwagger* files, and regenerate the package when there is a change." @@ -63,8 +63,6 @@ SELF_DIR=$(dirname $(realpath $0)) SELF_DIR=${SELF_DIR%/*} # remove the last directory (tools) from the path cd $SELF_DIR -SWAGGER_YAML=$SELF_DIR/src/KaggleSwagger.yaml -SWAGGER_CONFIG=$SELF_DIR/src/KaggleSwaggerConfig.json KAGGLE_XDG_CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/kaggle" mkdir -p "$KAGGLE_XDG_CONFIG_DIR" KAGGLE_DEV_CONFIG_DIR=$(realpath "$KAGGLE_XDG_CONFIG_DIR/dev") @@ -109,31 +107,6 @@ function create-local-creds { chmod 600 $kaggle_config_file } -function generate-package { - # TODO Remove this. - if [[ -f "kaggle/api/__init__.py" ]]; then - sed -i -e 's/kaggle_api/kaggle_api_extended/g' kaggle/api/__init__.py - fi - - if [[ -f "kaggle/configuration.py" ]]; then - # Replace the hard-coded endpoint URL with an environment variable in configuration.py - # to allow talking to localhost, staging, prod etc. - sed -i 's|self.host = "http|self.host = _get_endpoint_from_env() or "http|g' kaggle/configuration.py - echo -e "\n" >> kaggle/configuration.py - cat <<-END >> kaggle/configuration.py -def _get_endpoint_from_env(): - import os - endpoint = os.environ.get("KAGGLE_API_ENDPOINT") - if endpoint is None: - return None - endpoint = endpoint.rstrip("/") - if endpoint.endswith("/api/v1"): - return endpoint - return endpoint + "/api/v1" -END - fi -} - function copy-src { cp ./src/setup.py . cp ./src/setup.cfg . @@ -151,16 +124,19 @@ function run-tests { fi if [[ "$TEST" == "$LOCAL_ENV" ]]; then - source ../use-localhost.sh + source tools/use-localhost.sh elif [[ "$TEST" == "$PROD_ENV" ]]; then - source ../use-prod.sh + source tools/use-prod.sh else return 0 # Nothing to do fi - cp ../python_api_tests.py ../sample_submission.csv ./ - python3 python_api_tests.py - rm -f ./python_api_tests.py + cd tests + ln -s ../kagglesdk . + ln -s ../kaggle . + python3 unit_tests.py + rm kaggle kagglesdk + cd .. } function install-package { @@ -177,10 +153,8 @@ function cleanup { rm -rf tox.ini \ test-requirements.txt \ test \ - .swagger-codegen \ .travis.yml \ git_push.sh \ - python_api_tests.py \ sample_submission.csv \ ds_salaries.csv \ test.csv \ @@ -194,7 +168,6 @@ function cleanup { function run { reset - generate-package copy-src run-autogen install-package @@ -205,28 +178,11 @@ function run { WATCHED_EVENTS="-e create -e modify -e delete" -function watch-swagger { - local watched_paths="$SWAGGER_YAML $SWAGGER_CONFIG" - - echo "Watching for changes to Swagger config..." - while inotifywait -q -r $WATCHED_EVENTS --format "%e %w%f" $watched_paths; do - echo "Deleting $SELF_DIR/kaggle/ $SELF_DIR/kaggle/" - rm -rf $SELF_DIR/kaggle/* - generate-package - run-autogen - copy-src - echo -e "\nWatching for changes to Swagger config..." - done -} - function watch-src { local watched_paths="$SELF_DIR/src" echo "Watching for changes under \"src\"..." while inotifywait -q -r $WATCHED_EVENTS --format "%e %w%f" $watched_paths; do - # Do not delete the output directory when there is no Swagger change to avoid - # having to run generate-package for each small code change as Swagger code - # generation is a bit slow (can take 2-3 seconds). echo "Copying changes..." copy-src echo "Done!" @@ -244,9 +200,8 @@ function watch { TEST="no" echo - watch-swagger & - local pid=$! watch-src + local pid=$! wait $pid }