From 8d00db02bc5c62cf2b4f1367dbef4f4bc0c89faa Mon Sep 17 00:00:00 2001 From: Manuel Date: Wed, 28 Apr 2021 23:00:51 +0200 Subject: [PATCH 001/547] Adding pandarallel library (#1005) * Adding pandarallel library Pandarallel allows to execute pandas apply method in parallel, which allows to do data preprocessing faster and easier. This is very he;pful in kernel only competiotions. See: https://github.com/nalepae/pandarallel * Adding test to pandarallel Adding a simple test --- Dockerfile | 1 + tests/test_pandarralel.py | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/test_pandarralel.py diff --git a/Dockerfile b/Dockerfile index 21c8e975..6bd15db6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -427,6 +427,7 @@ RUN pip install flashtext && \ pip install jax==0.2.12 jaxlib==0.1.64 && \ # ipympl adds interactive widget support for matplotlib pip install ipympl==0.7.0 && \ + pip install pandarallel && \ /tmp/clean-layer.sh # Download base easyocr models. diff --git a/tests/test_pandarralel.py b/tests/test_pandarralel.py new file mode 100644 index 00000000..fe74b0e1 --- /dev/null +++ b/tests/test_pandarralel.py @@ -0,0 +1,11 @@ +import unittest + +import pandas as pd +from pandarallel import pandarallel + +pandarallel.initialize() + +class TestPandarallel(unittest.TestCase): + def test_pandarallel(self): + data = pd.read_csv("/input/tests/data/train.csv") + data['label_converted'] = data['label'].parallel_apply(lambda x: x+1) From 7f983e2cca1863d41631edaffddea7c6fab986ad Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 6 May 2021 12:13:41 -0700 Subject: [PATCH 002/547] Fix spacy test. (#1008) spacy has been upgraded to 3.x (fastai had a 2.x requirement but now support 3.x which upgraded spacy.) --- Dockerfile | 2 +- tests/test_spacy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6bd15db6..066e6494 100644 --- a/Dockerfile +++ b/Dockerfile @@ -350,7 +350,7 @@ RUN pip install bleach && \ pip install allennlp && \ # https://b.corp.google.com/issues/184685619#comment9: 3.9.0 is causing a major performance degradation with spacy 2.3.5 pip install importlib-metadata==3.4.0 && \ - python -m spacy download en && python -m spacy download en_core_web_lg && \ + python -m spacy download en_core_web_sm && python -m spacy download en_core_web_lg && \ apt-get install -y ffmpeg && \ /tmp/clean-layer.sh diff --git a/tests/test_spacy.py b/tests/test_spacy.py index bd58cc30..22a44c15 100644 --- a/tests/test_spacy.py +++ b/tests/test_spacy.py @@ -4,6 +4,6 @@ class TestSpacy(unittest.TestCase): def test_model(self): - nlp = spacy.load('en') + nlp = spacy.load('en_core_web_sm') doc = nlp('This is a sentence.') self.assertEqual(5, len(doc)) From c287f51ffa498f0a49ed65193169fdcb8da31893 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 7 May 2021 10:17:06 -0700 Subject: [PATCH 003/547] Fix warning in openslide test (#1009) ``` ResourceWarning: unclosed file <_io.BufferedReader name='/input/tests/data/test.tif'> testMethod() ResourceWarning: Enable tracemalloc to get the object allocation traceback ``` --- tests/test_openslide.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_openslide.py b/tests/test_openslide.py index 6524ef40..f0d7c5b5 100644 --- a/tests/test_openslide.py +++ b/tests/test_openslide.py @@ -4,6 +4,5 @@ class TestOpenslide(unittest.TestCase): def test_read_tif(self): - slide = open_slide('/input/tests/data/test.tif') - - self.assertEqual(1, slide.level_count) + with open_slide('/input/tests/data/test.tif') as slide: + self.assertEqual(1, slide.level_count) From 8abf69a509f3e42dc791c243b3a7d81882a436bf Mon Sep 17 00:00:00 2001 From: Philippe Modard Date: Thu, 13 May 2021 14:47:23 -0400 Subject: [PATCH 004/547] Fix diff output (#1010) http://b/178087631 --- diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diff b/diff index cd0fc732..79b9b756 100755 --- a/diff +++ b/diff @@ -94,7 +94,7 @@ if [[ -n "$PACKAGE_NAME" ]]; then echo "Package: $PACKAGE_NAME" CMDS=("python /tools/pip_list_versions.py $PACKAGE_NAME | sort") else - CMDS=("pip freeze" 'cat /etc/os-release | grep -oP "PRETTY_NAME=\"\K([^\"]*)"' "uname -r" "dpkg --list | awk '{print \$2\"==\"\$3}'") + CMDS=("pip list --format=freeze" 'cat /etc/os-release | grep -oP "PRETTY_NAME=\"\K([^\"]*)"' "uname -r" "dpkg --list | awk '{print \$2\"==\"\$3}'") fi for cmd in "${CMDS[@]}"; do From ef0e8b931537a6d3b5a0f820a2232d597d89a4ad Mon Sep 17 00:00:00 2001 From: Bovard Doerschuk-Tiberi Date: Fri, 21 May 2021 08:53:15 -0700 Subject: [PATCH 005/547] Fixed ucaip integration to work with latest code --- patches/kaggle_gcp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index 07179fa9..466f17fc 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -198,7 +198,7 @@ def patched_init(self, *args, **kwargs): if specified_credentials is None: Log.info("No credentials specified, using KaggleKernelCredentials.") kwargs['credentials'] = kaggle_kernel_credentials - return client_init(self, *args, **kwargs) + return client_init(self, *args, **kwargs) if (not has_been_monkeypatched(client_klass.init)): client_klass.init = patched_init @@ -340,7 +340,7 @@ def init_ucaip(): kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) # Patch the ucaip init method, this flows down to all ucaip services - monkeypatch_init(aiplatform.initializer.global_config, kaggle_kernel_credentials) + monkeypatch_init(aiplatform, kaggle_kernel_credentials) def init_video_intelligence(): from google.cloud import videointelligence From 16756501205ff2631d2cd4c1e19f3ae548646815 Mon Sep 17 00:00:00 2001 From: Bovard Doerschuk-Tiberi Date: Fri, 21 May 2021 08:57:55 -0700 Subject: [PATCH 006/547] Better comments and variable names --- patches/kaggle_gcp.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index 466f17fc..66181613 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -187,22 +187,22 @@ def monkeypatch_bq(bq_client, *args, **kwargs): bq_client, *args, **kwargs) return bigquery -# Monkey patch classes that use the init method +# Monkey patch for aiplatform init # eg # from google.cloud import aiplatform # aiplatform.init(args) -def monkeypatch_init(client_klass, kaggle_kernel_credentials): - client_init = client_klass.init +def monkeypatch_aiploatform_init(aiplatform_klass, kaggle_kernel_credentials): + aiplatform_init = aiplatform_klass.init def patched_init(self, *args, **kwargs): specified_credentials = kwargs.get('credentials') if specified_credentials is None: Log.info("No credentials specified, using KaggleKernelCredentials.") kwargs['credentials'] = kaggle_kernel_credentials - return client_init(self, *args, **kwargs) + return aiplatform_init(self, *args, **kwargs) - if (not has_been_monkeypatched(client_klass.init)): - client_klass.init = patched_init - Log.info(f"Client patched: {client_klass}") + if (not has_been_monkeypatched(aiplatform_klass.init)): + aiplatform_klass.init = patched_init + Log.info("aiplatform.init patched") def monkeypatch_client(client_klass, kaggle_kernel_credentials): client_init = client_klass.__init__ @@ -340,7 +340,7 @@ def init_ucaip(): kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) # Patch the ucaip init method, this flows down to all ucaip services - monkeypatch_init(aiplatform, kaggle_kernel_credentials) + monkeypatch_aiplatform_init(aiplatform, kaggle_kernel_credentials) def init_video_intelligence(): from google.cloud import videointelligence From 5171b1d639251b41060cc93cb34f410af5a29845 Mon Sep 17 00:00:00 2001 From: Bovard Doerschuk-Tiberi Date: Fri, 21 May 2021 09:46:42 -0700 Subject: [PATCH 007/547] fix typo --- patches/kaggle_gcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index 66181613..fa3aae5b 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -191,7 +191,7 @@ def monkeypatch_bq(bq_client, *args, **kwargs): # eg # from google.cloud import aiplatform # aiplatform.init(args) -def monkeypatch_aiploatform_init(aiplatform_klass, kaggle_kernel_credentials): +def monkeypatch_aiplatform_init(aiplatform_klass, kaggle_kernel_credentials): aiplatform_init = aiplatform_klass.init def patched_init(self, *args, **kwargs): specified_credentials = kwargs.get('credentials') From 597bc2a20013bc6f93da1ec8dd3606ff675ed580 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 25 May 2021 10:58:09 -0700 Subject: [PATCH 008/547] Pin fastai to 2.2.7 (#1014) fastai 2.3.1 upgrades torch and torchvision causing issues. upgrading torch proved more difficult than I thought (unresolvable conflicts atm). will handle the upgrade later in b/181966788 http://b/188429515 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 066e6494..dde1ce04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -346,7 +346,8 @@ RUN pip install bleach && \ pip install widgetsnbextension && \ pip install pyarrow && \ pip install feather-format && \ - pip install fastai && \ + # fastai >= 2.3.1 upgrades pytorch/torchvision. upgrade of pytorch will be handled in b/181966788 + pip install fastai==2.2.7 && \ pip install allennlp && \ # https://b.corp.google.com/issues/184685619#comment9: 3.9.0 is causing a major performance degradation with spacy 2.3.5 pip install importlib-metadata==3.4.0 && \ From 97ab4b8de203f4856cf5bcb7960419ad2b982c39 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 25 May 2021 11:31:54 -0700 Subject: [PATCH 009/547] Fix pytorch lightning test. (#1019) In 1.3.x, trainer.fit(...) returns None. Asserting on trainer.logged_metrics instead. http://b/188429515 --- tests/test_pytorch_lightning.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_pytorch_lightning.py b/tests/test_pytorch_lightning.py index a7c89d85..24d9f16d 100644 --- a/tests/test_pytorch_lightning.py +++ b/tests/test_pytorch_lightning.py @@ -23,10 +23,10 @@ def setup(self, stage=None): self.valid_ds = TensorDataset(X_valid, y_valid) def train_dataloader(self): - return DataLoader(self.train_ds, batch_size=self.batch_size, shuffle=True) + return DataLoader(self.train_ds, batch_size=self.batch_size, shuffle=True, num_workers=1) def val_dataloader(self): - return DataLoader(self.valid_ds, batch_size=self.batch_size, shuffle=False) + return DataLoader(self.valid_ds, batch_size=self.batch_size, shuffle=False, num_workers=1) class LitClassifier(pl.LightningModule): @@ -54,7 +54,6 @@ def validation_step(self, batch, batch_idx): def configure_optimizers(self): return torch.optim.Adam(self.parameters(), lr=1e-2) - class TestPytorchLightning(unittest.TestCase): def test_version(self): @@ -64,5 +63,8 @@ def test_mnist(self): dm = LitDataModule() model = LitClassifier() trainer = pl.Trainer(gpus=None, max_epochs=1) - result = trainer.fit(model, datamodule=dm) - self.assertTrue(result) + trainer.fit(model, datamodule=dm) + + self.assertIn("train_loss", trainer.logged_metrics) + self.assertIn("val_loss", trainer.logged_metrics) + From ce2a5245a1e2481e7384caec03feafe38bac24da Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 26 May 2021 09:28:42 -0700 Subject: [PATCH 010/547] Pin tensorflow-transform (#1020) `tensorflow-transform` 1.x released on May 24th is causing an unwanted upgrade to TensorFlow 2.5. http://b/188429515 --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index dde1ce04..edb93d7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -412,6 +412,8 @@ RUN pip install flashtext && \ pip install vowpalwabbit && \ # papermill can replace nbconvert for executing notebooks pip install cloud-tpu-client && \ + # b/188429515#comment7 tensorflow-transform 1.x is causing an upgrade to TensorFlow 2.5. + pip install tensorflow-transform==0.30.0 && \ pip install tensorflow-cloud && \ pip install tensorflow-datasets && \ pip install pydub && \ From 5be3502c23f2212e5c53fca33586cc212936f366 Mon Sep 17 00:00:00 2001 From: Philippe Modard Date: Wed, 26 May 2021 12:39:18 -0400 Subject: [PATCH 011/547] Main branch as default (#1021) http://b/188715321 --- Jenkinsfile | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 74b26dd9..8f0fe862 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : "" // Mon-Fri at noon UTC, 8am EST, 5am PDT +String cron_string = BRANCH_NAME == "main" ? "H 12 * * 1-5" : "" // Mon-Fri at noon UTC, 8am EST, 5am PDT pipeline { agent { label 'ephemeral-linux' } @@ -14,9 +14,9 @@ pipeline { GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() GIT_COMMIT_SUMMARY = "`` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" - SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim() - PRETEST_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"ci-pretest\"; else echo \"${GIT_BRANCH}-pretest\"; fi").trim() - STAGING_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"staging\"; else echo \"${GIT_BRANCH}-staging\"; fi").trim() + SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"main\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim() + PRETEST_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"main\" ]]; then echo \"ci-pretest\"; else echo \"${GIT_BRANCH}-pretest\"; fi").trim() + STAGING_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"main\" ]]; then echo \"staging\"; else echo \"${GIT_BRANCH}-staging\"; fi").trim() } stages { diff --git a/README.md b/README.md index 44be6c06..1c8481bf 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ If you the first step above doesn't work for your use case, [open an issue](http 1. For changes specific to the GPU image, update the [gpu.Dockerfile](gpu.Dockerfile). 1. Otherwise, update the [Dockerfile](Dockerfile). 1. Follow the instructions below to build a new image. -1. Add tests for your new package. See this [example](https://github.com/Kaggle/docker-python/blob/master/tests/test_fastai.py). +1. Add tests for your new package. See this [example](https://github.com/Kaggle/docker-python/blob/main/tests/test_fastai.py). 1. Follow the instructions below to test the new image. 1. Open a PR on this repo and you are all set! From 4e3c116f1fcb98db641717060a831cb67d124969 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 26 May 2021 13:06:51 -0700 Subject: [PATCH 012/547] Pin tensorflow-cloud to 1.1.13 (#1022) Starting in 1.1.14, `tensorflow-cloud` requires `tensorflow-transform` which requires `apache-beam[gcp]`. `apache-beam` downgrades the `google.cloud` libraries to 1.x. However, we are using 2.x http://b/188429515 --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index edb93d7c..4c7a3947 100644 --- a/Dockerfile +++ b/Dockerfile @@ -412,9 +412,8 @@ RUN pip install flashtext && \ pip install vowpalwabbit && \ # papermill can replace nbconvert for executing notebooks pip install cloud-tpu-client && \ - # b/188429515#comment7 tensorflow-transform 1.x is causing an upgrade to TensorFlow 2.5. - pip install tensorflow-transform==0.30.0 && \ - pip install tensorflow-cloud && \ + # b/188429515#comment7 tensorflow-cloud >= 0.1.14 installs tensorflow-transform which install apache-beam which downgrades the google.cloud library to 1.x. + pip install tensorflow-cloud==0.1.13 && \ pip install tensorflow-datasets && \ pip install pydub && \ pip install pydegensac && \ From 79b6541e84829fe3985421bc0b5ebfe8e34052c9 Mon Sep 17 00:00:00 2001 From: Bovard Doerschuk-Tiberi Date: Thu, 27 May 2021 10:10:17 -0700 Subject: [PATCH 013/547] Fix ucaip init method (#1024) --- patches/kaggle_gcp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index fa3aae5b..2c8b64cc 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -193,12 +193,12 @@ def monkeypatch_bq(bq_client, *args, **kwargs): # aiplatform.init(args) def monkeypatch_aiplatform_init(aiplatform_klass, kaggle_kernel_credentials): aiplatform_init = aiplatform_klass.init - def patched_init(self, *args, **kwargs): + def patched_init(*args, **kwargs): specified_credentials = kwargs.get('credentials') if specified_credentials is None: Log.info("No credentials specified, using KaggleKernelCredentials.") kwargs['credentials'] = kaggle_kernel_credentials - return aiplatform_init(self, *args, **kwargs) + return aiplatform_init(*args, **kwargs) if (not has_been_monkeypatched(aiplatform_klass.init)): aiplatform_klass.init = patched_init From e2ec76e9bd7612283b55eb232d1ec7d0b1abe14b Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 9 Jun 2021 13:47:59 -0700 Subject: [PATCH 014/547] Pin pynvml (#1027) The latest version is causing an issue with nnabla. nnabla fixed the issue, pinning while waiting for new version of nnabla: https://github.com/sony/nnabla/issues/892 http://b/190622765 --- gpu.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index b40d0807..46a725b1 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -100,6 +100,8 @@ RUN pip install /tmp/tfa_gpu/tensorflow*.whl && \ RUN pip install pycuda && \ pip install cupy-cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION && \ pip install pynvrtc && \ + # b/190622765 latest version is causing issue. nnabla fixed it in https://github.com/sony/nnabla/issues/892, waiting for new release before we can remove this pin. + pip install pynvml==8.0.4 && \ pip install nnabla-ext-cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION && \ /tmp/clean-layer.sh From e8cb8a2391909fb609ea007ffc877c53dbcfc188 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 9 Jun 2021 22:12:13 -0700 Subject: [PATCH 015/547] Upgrade base image to m71 (#1028) --- Dockerfile | 2 +- tensorflow-whl/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4c7a3947..121adfcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_TAG=m66 +ARG BASE_TAG=m71 ARG TENSORFLOW_VERSION=2.4.1 FROM gcr.io/kaggle-images/python-tensorflow-whl:${TENSORFLOW_VERSION}-py37-2 as tensorflow_whl diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile index e49314af..9d543d73 100644 --- a/tensorflow-whl/Dockerfile +++ b/tensorflow-whl/Dockerfile @@ -1,5 +1,5 @@ FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 AS nvidia -FROM gcr.io/deeplearning-platform-release/base-cpu:m66 +FROM gcr.io/deeplearning-platform-release/base-cpu:m71 # Avoid interactive configuration prompts/dialogs during apt-get. ENV DEBIAN_FRONTEND=noninteractive From 3f0701e8c973646775f8fa25c888254f717b2b7b Mon Sep 17 00:00:00 2001 From: Alexis Cook Date: Thu, 10 Jun 2021 12:12:41 -0400 Subject: [PATCH 016/547] Update test_spacy.py (#1026) --- tests/test_spacy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_spacy.py b/tests/test_spacy.py index 22a44c15..ae524dae 100644 --- a/tests/test_spacy.py +++ b/tests/test_spacy.py @@ -2,6 +2,7 @@ import spacy +# please check with learn team before changing class TestSpacy(unittest.TestCase): def test_model(self): nlp = spacy.load('en_core_web_sm') From 6d08285a05dcb45df3370959d43c1452dff9c4ba Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 10 Jun 2021 09:26:26 -0700 Subject: [PATCH 017/547] Add igraph test (#1029) --- tests/test_igraph.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/test_igraph.py diff --git a/tests/test_igraph.py b/tests/test_igraph.py new file mode 100644 index 00000000..67cdbbbd --- /dev/null +++ b/tests/test_igraph.py @@ -0,0 +1,12 @@ +import unittest + +from igraph import Graph + + +class TestIgraph(unittest.TestCase): + def test_graph(self): + # Create a graph with 10 vertices & 2 children each. + g2 = Graph.Tree(n=10, children=2) + + self.assertEqual(9, len(g2.get_edgelist())) + From 411e2f50ebc94f8485bd0c9848e727d51288f5e4 Mon Sep 17 00:00:00 2001 From: Jovan Veljanoski Date: Fri, 18 Jun 2021 15:29:07 +0200 Subject: [PATCH 018/547] Add the vaex library (includes a test) (#1031) --- Dockerfile | 1 + tests/test_vaex.py | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/test_vaex.py diff --git a/Dockerfile b/Dockerfile index 121adfcb..8a034ba6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -365,6 +365,7 @@ RUN pip install bleach && \ RUN pip install flashtext && \ pip install wandb && \ + pip install vaex && \ pip install marisa-trie && \ pip install pyemd && \ pip install pyupset && \ diff --git a/tests/test_vaex.py b/tests/test_vaex.py new file mode 100644 index 00000000..db5b9aba --- /dev/null +++ b/tests/test_vaex.py @@ -0,0 +1,10 @@ +import unittest + +import vaex + +class TestVaex(unittest.TestCase): + def test_read_csv(self): + df = vaex.read_csv("/input/tests/data/train.csv") + + self.assertEqual((100, 785), df.shape) + self.assertEqual(10, df['label'].nunique()) From 8c40f18e7e70b252385ce8d05e706bd258fa7167 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 21 Jun 2021 15:16:30 -0700 Subject: [PATCH 019/547] Revert "Add the vaex library (includes a test) (#1031)" (#1033) This reverts commit 411e2f50ebc94f8485bd0c9848e727d51288f5e4. --- Dockerfile | 1 - tests/test_vaex.py | 10 ---------- 2 files changed, 11 deletions(-) delete mode 100644 tests/test_vaex.py diff --git a/Dockerfile b/Dockerfile index 8a034ba6..121adfcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -365,7 +365,6 @@ RUN pip install bleach && \ RUN pip install flashtext && \ pip install wandb && \ - pip install vaex && \ pip install marisa-trie && \ pip install pyemd && \ pip install pyupset && \ diff --git a/tests/test_vaex.py b/tests/test_vaex.py deleted file mode 100644 index db5b9aba..00000000 --- a/tests/test_vaex.py +++ /dev/null @@ -1,10 +0,0 @@ -import unittest - -import vaex - -class TestVaex(unittest.TestCase): - def test_read_csv(self): - df = vaex.read_csv("/input/tests/data/train.csv") - - self.assertEqual((100, 785), df.shape) - self.assertEqual(10, df['label'].nunique()) From 2dbd486fc9ce05aa5b4fac0cb47373ab228baae7 Mon Sep 17 00:00:00 2001 From: Jim Plotts Date: Mon, 21 Jun 2021 19:38:48 -0400 Subject: [PATCH 020/547] Pin fancyimpute to prevent tensorflow reinstallation. (#1032) --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 121adfcb..aabaa814 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,7 +194,8 @@ RUN pip install mpld3 && \ pip install CVXcanon && \ # b/179264579 cvxpy 1.1.8 requires numpy >= 1.20 pip install cvxpy==1.1.7 && \ - pip install fancyimpute && \ + # b/184083722 remove pin after upgrading to TensorFlow 2.5 + pip install fancyimpute==0.5.5 && \ pip install pymc3 && \ pip install imagecodecs && \ pip install tifffile && \ From 0b2dcfa233cd97f058a9d2515637ae61c3a0b026 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 22 Jun 2021 13:46:47 -0700 Subject: [PATCH 021/547] Upgrade base image (#1034) --- Dockerfile | 2 +- tensorflow-whl/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index aabaa814..679f591d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_TAG=m71 +ARG BASE_TAG=m73 ARG TENSORFLOW_VERSION=2.4.1 FROM gcr.io/kaggle-images/python-tensorflow-whl:${TENSORFLOW_VERSION}-py37-2 as tensorflow_whl diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile index 9d543d73..0cc4140a 100644 --- a/tensorflow-whl/Dockerfile +++ b/tensorflow-whl/Dockerfile @@ -1,5 +1,5 @@ FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 AS nvidia -FROM gcr.io/deeplearning-platform-release/base-cpu:m71 +FROM gcr.io/deeplearning-platform-release/base-cpu:m73 # Avoid interactive configuration prompts/dialogs during apt-get. ENV DEBIAN_FRONTEND=noninteractive From 3d698a638975d175594c8e352fba3bee7d8a3020 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 25 Jun 2021 13:22:39 -0700 Subject: [PATCH 022/547] Use official TensorFlow wheel (#1036) We are moving towards the base DLVM TF image. http://b/186646069 --- Dockerfile | 27 +++++++-------------------- gpu.Dockerfile | 11 +---------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 679f591d..f30fc2c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,12 @@ ARG BASE_TAG=m73 ARG TENSORFLOW_VERSION=2.4.1 -FROM gcr.io/kaggle-images/python-tensorflow-whl:${TENSORFLOW_VERSION}-py37-2 as tensorflow_whl FROM gcr.io/deeplearning-platform-release/base-cpu:${BASE_TAG} +# We need to redefine TENSORFLOW_VERSION here to get the default ARG value defined above the FROM instruction. +# See: https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG TENSORFLOW_VERSION + ADD clean-layer.sh /tmp/clean-layer.sh ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl ADD patches/template_conf.json /opt/kaggle/conf.json @@ -56,22 +59,9 @@ RUN pip install seaborn python-dateutil dask && \ pip install -f https://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o && \ /tmp/clean-layer.sh -# Install tensorflow from a pre-built wheel -COPY --from=tensorflow_whl /tmp/tensorflow_cpu/*.whl /tmp/tensorflow_cpu/ -RUN pip install /tmp/tensorflow_cpu/tensorflow*.whl && \ - rm -rf /tmp/tensorflow_cpu && \ - /tmp/clean-layer.sh - -# Install tensorflow-gcs-config from a pre-built wheel -COPY --from=tensorflow_whl /tmp/tensorflow_gcs_config/*.whl /tmp/tensorflow_gcs_config/ -RUN pip install /tmp/tensorflow_gcs_config/tensorflow*.whl && \ - rm -rf /tmp/tensorflow_gcs_config && \ - /tmp/clean-layer.sh - -# Install TensorFlow addons (TFA). -COPY --from=tensorflow_whl /tmp/tfa_cpu/*.whl /tmp/tfa_cpu/ -RUN pip install /tmp/tfa_cpu/tensorflow*.whl && \ - rm -rf /tmp/tfa_cpu/ && \ +RUN pip install tensorflow==${TENSORFLOW_VERSION} && \ + pip install tensorflow-gcs-config==2.4.0 && \ + pip install tensorflow-addons==0.12.1 && \ /tmp/clean-layer.sh RUN apt-get install -y libfreetype6-dev && \ @@ -513,9 +503,6 @@ RUN jupyter-nbextension disable nb_conda --py --sys-prefix && \ # Set backend for matplotlib ENV MPLBACKEND "agg" -# We need to redefine TENSORFLOW_VERSION here to get the default ARG value defined above the FROM instruction. -# See: https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact -ARG TENSORFLOW_VERSION ARG GIT_COMMIT=unknown ARG BUILD_DATE=unknown diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 46a725b1..42ac20c9 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -1,7 +1,6 @@ ARG BASE_TAG=staging FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 AS nvidia -FROM gcr.io/kaggle-images/python-tensorflow-whl:2.4.1-py37-2 as tensorflow_whl FROM gcr.io/kaggle-images/python:${BASE_TAG} ADD clean-layer.sh /tmp/clean-layer.sh @@ -82,20 +81,12 @@ RUN pip install jax==0.2.12 jaxlib==0.1.64+cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_V /tmp/clean-layer.sh # Reinstall packages with a separate version for GPU support. -COPY --from=tensorflow_whl /tmp/tensorflow_gpu/*.whl /tmp/tensorflow_gpu/ RUN pip uninstall -y tensorflow && \ - pip install /tmp/tensorflow_gpu/tensorflow*.whl && \ - rm -rf /tmp/tensorflow_gpu && \ + pip install tensorflow-gpu==2.4.1 && \ pip uninstall -y mxnet && \ pip install mxnet-cu$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION && \ /tmp/clean-layer.sh - # Reinstall TensorFlow addons (TFA) with GPU support. -COPY --from=tensorflow_whl /tmp/tfa_gpu/*.whl /tmp/tfa_gpu/ -RUN pip install /tmp/tfa_gpu/tensorflow*.whl && \ - rm -rf /tmp/tfa_gpu/ && \ - /tmp/clean-layer.sh - # Install GPU-only packages RUN pip install pycuda && \ pip install cupy-cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION && \ From 391dc1b8205b1fa337781b78f82c58ae67b441da Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 2 Jul 2021 09:07:10 -0700 Subject: [PATCH 023/547] Upgrade cuml/cudf to 21.06 (#1038) * Upgrade cuml/cudf to 21.06 http://b/192568337 * add torchtext to list of package to remove to avoid inconsistent environment * Use cupy installed by conda --- gpu.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 42ac20c9..fbce4ede 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -57,8 +57,9 @@ RUN apt-get install -y ocl-icd-libopencl1 clinfo libboost-all-dev && \ # the remaining pip commands: https://www.anaconda.com/using-pip-in-a-conda-environment/ # However, because this image is based on the CPU image, this isn't possible but better # to put them at the top of this file to minize conflicts. -RUN conda remove --force -y pytorch torchvision torchaudio cpuonly && \ - conda install "pytorch=1.7" "torchvision=0.8" "torchaudio=0.7" "torchtext=0.8" "cudf=0.16" "cuml=0.16" cudatoolkit=$CUDA_VERSION && \ +RUN conda remove --force -y pytorch torchvision torchaudio torchtext cpuonly && \ + conda install "pytorch=1.7" "torchvision=0.8" "torchaudio=0.7" "torchtext=0.8" cudatoolkit=$CUDA_VERSION && \ + conda install "cudf=21.06" "cuml=21.06" && \ /tmp/clean-layer.sh # Install LightGBM with GPU @@ -89,7 +90,6 @@ RUN pip uninstall -y tensorflow && \ # Install GPU-only packages RUN pip install pycuda && \ - pip install cupy-cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION && \ pip install pynvrtc && \ # b/190622765 latest version is causing issue. nnabla fixed it in https://github.com/sony/nnabla/issues/892, waiting for new release before we can remove this pin. pip install pynvml==8.0.4 && \ From 7e3c7a28423aa3609c6040b5b3e834157245c2c2 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 2 Jul 2021 09:07:29 -0700 Subject: [PATCH 024/547] Upgrade LightGBM 3.2.1 (#1040) --- Dockerfile | 2 +- gpu.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f30fc2c0..7cf1c06f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,7 @@ RUN apt-get install -y libfreetype6-dev && \ pip install wordcloud && \ pip install xgboost && \ # Pinned to match GPU version. Update version together. - pip install lightgbm==3.2.0 && \ + pip install lightgbm==3.2.1 && \ pip install pydot && \ pip install keras && \ pip install keras-tuner && \ diff --git a/gpu.Dockerfile b/gpu.Dockerfile index fbce4ede..62fc92fc 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -67,7 +67,7 @@ RUN pip uninstall -y lightgbm && \ cd /usr/local/src && \ git clone --recursive https://github.com/microsoft/LightGBM && \ cd LightGBM && \ - git checkout tags/v3.2.0 && \ + git checkout tags/v3.2.1 && \ mkdir build && cd build && \ cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. && \ make -j$(nproc) && \ From 882546e94435036c068442f04638a7346745c7b5 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 2 Jul 2021 09:07:41 -0700 Subject: [PATCH 025/547] Upgrade JAX to 0.2.16 (#1039) --- gpu.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 62fc92fc..e467089a 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -78,8 +78,8 @@ RUN pip uninstall -y lightgbm && \ /tmp/clean-layer.sh # Install JAX (Keep JAX version in sync with CPU image) -RUN pip install jax==0.2.12 jaxlib==0.1.64+cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION -f https://storage.googleapis.com/jax-releases/jax_releases.html && \ - /tmp/clean-layer.sh +RUN pip install jax==0.2.16 jaxlib==0.1.68+cuda$CUDA_MAJOR_VERSION$CUDA_MINOR_VERSION -f https://storage.googleapis.com/jax-releases/jax_releases.html && \ + /tmp/clean-layer.sh # Reinstall packages with a separate version for GPU support. RUN pip uninstall -y tensorflow && \ From 6cd8466003904aab238c9336238ef4eb60a18aa0 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 9 Jul 2021 13:22:16 -0700 Subject: [PATCH 026/547] Removing fancyimpute (#1042) * Removing fancyimpute This package is in "bare maintenance" mode and rarely used on Kaggle. `cvxpy` was only used by the `fancyimpute` package. * pyldavis >= 3.3 requires numpy >= 1.20.0 but TensorFlow 2.4.1 / 2.5.0 requires 1.19.2 --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7cf1c06f..40237d0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -174,7 +174,8 @@ RUN pip install mpld3 && \ pip install s2sphere && \ pip install bayesian-optimization && \ pip install matplotlib-venn && \ - pip install pyldavis && \ + # b/184083722 pyldavis >= 3.3 requires numpy >= 1.20.0 but TensorFlow 2.4.1 / 2.5.0 requires 1.19.2 + pip install pyldavis==3.2.2 && \ pip install mlxtend && \ pip install altair && \ # b/183944405 pystan 3.x is not compatible with fbprophet. @@ -182,10 +183,6 @@ RUN pip install mpld3 && \ pip install ImageHash && \ pip install ecos && \ pip install CVXcanon && \ - # b/179264579 cvxpy 1.1.8 requires numpy >= 1.20 - pip install cvxpy==1.1.7 && \ - # b/184083722 remove pin after upgrading to TensorFlow 2.5 - pip install fancyimpute==0.5.5 && \ pip install pymc3 && \ pip install imagecodecs && \ pip install tifffile && \ From 0638abff8540820218904a1aa67f93ae6a5fadbd Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 12 Jul 2021 13:44:03 -0700 Subject: [PATCH 027/547] Remove multi-backend Keras. (#1045) From the keras repo: https://github.com/keras-team/keras/tree/2.4.0 > Multi-backend Keras has been discontinued. At this time, we recommend that Keras users who use multi-backend Keras with the TensorFlow backend switch to tf.keras in TensorFlow 2.0. http://b/184083722 --- Dockerfile | 7 +--- tests/test_keras.py | 82 --------------------------------------------- 2 files changed, 1 insertion(+), 88 deletions(-) delete mode 100644 tests/test_keras.py diff --git a/Dockerfile b/Dockerfile index 40237d0e..f24e6428 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,6 @@ RUN apt-get install -y libfreetype6-dev && \ # Pinned to match GPU version. Update version together. pip install lightgbm==3.2.1 && \ pip install pydot && \ - pip install keras && \ pip install keras-tuner && \ pip install flake8 && \ # Pinned because it breaks theano test with the latest version (b/178107003). @@ -248,10 +247,6 @@ RUN pip install tensorpack && \ RUN pip install --upgrade cython && \ pip install --upgrade cysignals && \ pip install pyfasttext && \ - # ktext has an explicit dependency on Keras 2.2.4 which is not - # compatible with TensorFlow 2.0 (support was added in Keras 2.3.0). - # Add the package back once it is fixed upstream. - # pip install ktext && \ pip install fasttext && \ apt-get install -y libhunspell-dev && pip install hunspell && \ pip install annoy && \ @@ -450,7 +445,7 @@ ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ ENV MKL_THREADING_LAYER=GNU # Temporary fixes and patches - # Temporary patch for Dask getting downgraded, which breaks Keras +# Temporary patch for Dask getting downgraded, which breaks Keras RUN pip install --upgrade dask && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ diff --git a/tests/test_keras.py b/tests/test_keras.py deleted file mode 100644 index de57de27..00000000 --- a/tests/test_keras.py +++ /dev/null @@ -1,82 +0,0 @@ -import unittest - -import keras -import numpy as np -import pandas as pd - -from keras.models import Sequential -from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, LSTM -from keras.optimizers import RMSprop, SGD -from keras.utils.np_utils import to_categorical - -from common import gpu_test - -class TestKeras(unittest.TestCase): - def test_train(self): - train = pd.read_csv("/input/tests/data/train.csv") - - x_train = train.iloc[:,1:].values.astype('float32') - y_train = to_categorical(train.iloc[:,0].astype('int32')) - - model = Sequential() - model.add(Dense(units=10, input_dim=784, activation='softmax')) - - model.compile( - loss='categorical_crossentropy', - optimizer=RMSprop(lr=0.001), - metrics=['accuracy']) - - model.fit(x_train, y_train, epochs=1, batch_size=32) - - # Uses convnet which depends on libcudnn when running on GPU - def test_conv2d(self): - # Generate dummy data - x_train = np.random.random((100, 100, 100, 3)) - y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) - x_test = np.random.random((20, 100, 100, 3)) - y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10) - - model = Sequential() - # input: 100x100 images with 3 channels -> (100, 100, 3) tensors. - # this applies 32 convolution filters of size 3x3 each. - model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(100, 100, 3))) - model.add(Conv2D(32, (3, 3), activation='relu')) - model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Dropout(0.25)) - - model.add(Conv2D(64, (3, 3), activation='relu')) - model.add(Conv2D(64, (3, 3), activation='relu')) - model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Dropout(0.25)) - - model.add(Flatten()) - model.add(Dense(256, activation='relu')) - model.add(Dropout(0.5)) - model.add(Dense(10, activation='softmax')) - - sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) - - # This throws if libcudnn is not properly installed with on a GPU - model.compile(loss='categorical_crossentropy', optimizer=sgd) - model.fit(x_train, y_train, batch_size=32, epochs=1) - - model.evaluate(x_test, y_test, batch_size=32) - - def test_lstm(self): - x_train = np.random.random((100, 100, 100)) - y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) - x_test = np.random.random((20, 100, 100)) - y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10) - - sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) - - model = Sequential() - model.add(LSTM(32, return_sequences=True, input_shape=(100, 100))) - model.add(Flatten()) - model.add(Dense(10, activation='softmax')) - - - model.compile(loss='categorical_crossentropy', optimizer=sgd) - model.fit(x_train, y_train, batch_size=32, epochs=1) - model.evaluate(x_test, y_test, batch_size=32) - From 08e5a8a02344bd14c00fd1add8816384400dfc18 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 19 Jul 2021 09:21:19 -0700 Subject: [PATCH 028/547] Add tests for torchtext and torchvision (#1046) --- tests/test_torchtext.py | 12 ++++++++++++ tests/test_torchvision.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/test_torchtext.py create mode 100644 tests/test_torchvision.py diff --git a/tests/test_torchtext.py b/tests/test_torchtext.py new file mode 100644 index 00000000..f9fbf76f --- /dev/null +++ b/tests/test_torchtext.py @@ -0,0 +1,12 @@ +import unittest + +from torchtext.data.metrics import bleu_score + + +class TestTorchtext(unittest.TestCase): + def test_bleu_score(self): + candidate = [['I', 'love', 'Kaggle', 'Notebooks']] + refs = [[['Completely', 'Different']]] + + self.assertEqual(0, bleu_score(candidate, refs)) + diff --git a/tests/test_torchvision.py b/tests/test_torchvision.py new file mode 100644 index 00000000..c66688e0 --- /dev/null +++ b/tests/test_torchvision.py @@ -0,0 +1,25 @@ +import unittest + +import torch +import torchvision.transforms as transforms +import torchvision.transforms.functional as F + + +class TestTorchvision(unittest.TestCase): + def test_float_to_float(self): + input_dtype=torch.float32 + output_dtype=torch.float64 + input_image = torch.tensor((0.0, 1.0), dtype=input_dtype) + transform = transforms.ConvertImageDtype(output_dtype) + transform_script = torch.jit.script(F.convert_image_dtype) + + output_image = transform(input_image) + output_image_script = transform_script(input_image, output_dtype) + + # TODO(b/181966788) Uncomment after upgrade to pytorch 1.9.0 is done. + # torch.testing.assert_close(output_image_script, output_image, rtol=0.0, atol=1e-6) + + actual_min, actual_max = output_image.tolist() + + self.assertAlmostEqual(0, actual_min) + self.assertAlmostEqual(1, actual_max) \ No newline at end of file From 5246382a7ddf8501a1abe41be92fa9f1196a7122 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 19 Jul 2021 09:41:50 -0700 Subject: [PATCH 029/547] Upgrade JAX on CPU image to match GPU image. (#1047) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f24e6428..d05b0464 100644 --- a/Dockerfile +++ b/Dockerfile @@ -409,7 +409,7 @@ RUN pip install flashtext && \ pip install pycrypto && \ pip install easyocr && \ # Keep JAX version in sync with GPU image. - pip install jax==0.2.12 jaxlib==0.1.64 && \ + pip install jax==0.2.16 jaxlib==0.1.68 && \ # ipympl adds interactive widget support for matplotlib pip install ipympl==0.7.0 && \ pip install pandarallel && \ From a9223c7c1be24bd92f671644a2fae2617825991d Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 19 Jul 2021 18:00:15 -0700 Subject: [PATCH 030/547] Upgrade base image to m75 (#1048) * Upgrade base image to m75 * Loosen version restrictions - Remove version pin for pyproj - Use matplotlib version from base image - Install python-igraph from pip instead of conda --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d05b0464..8d4a5e90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_TAG=m73 +ARG BASE_TAG=m75 ARG TENSORFLOW_VERSION=2.4.1 FROM gcr.io/deeplearning-platform-release/base-cpu:${BASE_TAG} @@ -45,12 +45,12 @@ RUN conda config --add channels conda-forge && \ # ^ rapidsai is the highest priority channel, default lowest, conda-forge 2nd lowest. # b/182405233 pyproj 3.x is not compatible with basemap 1.2.1 # b/161473620#comment7 pin required to prevent resolver from picking pysal 1.x., pysal 2.2.x is also downloading data on import. - conda install matplotlib basemap cartopy python-igraph imagemagick "pyproj=2.6" "pysal==2.1.0" && \ + conda install basemap cartopy imagemagick pyproj "pysal==2.1.0" && \ conda install "pytorch=1.7" "torchvision=0.8" "torchaudio=0.7" "torchtext=0.8" cpuonly && \ /tmp/clean-layer.sh # The anaconda base image includes outdated versions of these packages. Update them to include the latest version. -RUN pip install seaborn python-dateutil dask && \ +RUN pip install seaborn python-dateutil dask python-igraph && \ pip install pyyaml joblib husl geopy ml_metrics mne pyshp && \ pip install pandas && \ # Install h2o from source. From 6d3f89dd9dad42a6576b82510e591d9bfc12001b Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 21 Jul 2021 09:11:52 -0700 Subject: [PATCH 031/547] Remove pytext-nlp (#1049) The latest version (0.3.3) was released in June 2020 and is only compatible with Pytorch 1.5: https://github.com/facebookresearch/pytext/blob/fad9639bf733ca0d5aaf2a3a9eaca64e90675d84/requirements.txt#L19 Usage on Kaggle is very low (~2-3 sessions per day). --- Dockerfile | 5 ----- tests/test_pytext.py | 14 -------------- 2 files changed, 19 deletions(-) delete mode 100644 tests/test_pytext.py diff --git a/Dockerfile b/Dockerfile index 8d4a5e90..0df911e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -370,11 +370,6 @@ RUN pip install flashtext && \ pip install https://github.com/hbasria/ggpy/archive/0.11.5.zip && \ pip install cesium && \ pip install rgf_python && \ - # b/185992410: onnx is a dependency of pytext, but the version 1.9.0 breaks pytext test. - # Remove this installation when pytext fixes the problem. - pip install onnx==1.8.1 && \ - # b/145404107: latest version force specific version of numpy and torch. - pip install pytext-nlp==0.1.2 && \ pip install tsfresh && \ pip install pykalman && \ pip install optuna && \ diff --git a/tests/test_pytext.py b/tests/test_pytext.py deleted file mode 100644 index 52d1c4bc..00000000 --- a/tests/test_pytext.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest - -from pytext.config.field_config import FeatureConfig -from pytext.data.featurizer import InputRecord, SimpleFeaturizer - -class TestPyText(unittest.TestCase): - - def test_tokenize(self): - featurizer = SimpleFeaturizer.from_config( - SimpleFeaturizer.Config(), FeatureConfig() - ) - - tokens = featurizer.featurize(InputRecord(raw_text="At eight o'clock")).tokens - self.assertEqual(['at', 'eight', "o'clock"], tokens) From 7eb00688a2878565716525f4c2886e0792024154 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 21 Jul 2021 17:35:11 -0700 Subject: [PATCH 032/547] Remove code to build custom TensorFlow wheel. (#1052) Starting in #1036, we are using the official TensorFlow wheels. --- README.md | 15 +- tensorflow-whl/CHANGELOG.md | 26 --- tensorflow-whl/Dockerfile | 131 ----------- tensorflow-whl/Jenkinsfile | 36 --- tensorflow-whl/README.md | 28 --- tensorflow-whl/build | 42 ---- tensorflow-whl/push | 48 ---- .../tensorflow-gcs-config/.gitignore | 12 - .../tensorflow-gcs-config/Dockerfile | 62 ----- .../tensorflow-gcs-config/MANIFEST.in | 2 - .../tensorflow-gcs-config/README.md | 10 - .../tensorflow-gcs-config/WORKSPACE | 44 ---- tensorflow-whl/tensorflow-gcs-config/build.py | 98 -------- tensorflow-whl/tensorflow-gcs-config/setup.py | 20 -- .../tensorflow_gcs_config/BUILD | 22 -- .../tensorflow_gcs_config/__init__.py | 132 ----------- .../gcs_config_op_kernels.cc | 218 ------------------ .../tensorflow_gcs_config/gcs_config_ops.cc | 66 ------ .../tensorflow-gcs-config/third_party/BUILD | 0 .../third_party/jsoncpp.BUILD | 37 --- .../third_party/tensorflow/BUILD | 0 .../third_party/tensorflow/BUILD.tpl | 18 -- .../third_party/tensorflow/tf_configure.bzl | 210 ----------------- 23 files changed, 1 insertion(+), 1276 deletions(-) delete mode 100644 tensorflow-whl/CHANGELOG.md delete mode 100644 tensorflow-whl/Dockerfile delete mode 100644 tensorflow-whl/Jenkinsfile delete mode 100644 tensorflow-whl/README.md delete mode 100755 tensorflow-whl/build delete mode 100755 tensorflow-whl/push delete mode 100644 tensorflow-whl/tensorflow-gcs-config/.gitignore delete mode 100644 tensorflow-whl/tensorflow-gcs-config/Dockerfile delete mode 100644 tensorflow-whl/tensorflow-gcs-config/MANIFEST.in delete mode 100644 tensorflow-whl/tensorflow-gcs-config/README.md delete mode 100644 tensorflow-whl/tensorflow-gcs-config/WORKSPACE delete mode 100644 tensorflow-whl/tensorflow-gcs-config/build.py delete mode 100644 tensorflow-whl/tensorflow-gcs-config/setup.py delete mode 100644 tensorflow-whl/tensorflow-gcs-config/tensorflow_gcs_config/BUILD delete mode 100644 tensorflow-whl/tensorflow-gcs-config/tensorflow_gcs_config/__init__.py delete mode 100644 tensorflow-whl/tensorflow-gcs-config/tensorflow_gcs_config/gcs_config_op_kernels.cc delete mode 100644 tensorflow-whl/tensorflow-gcs-config/tensorflow_gcs_config/gcs_config_ops.cc delete mode 100644 tensorflow-whl/tensorflow-gcs-config/third_party/BUILD delete mode 100644 tensorflow-whl/tensorflow-gcs-config/third_party/jsoncpp.BUILD delete mode 100644 tensorflow-whl/tensorflow-gcs-config/third_party/tensorflow/BUILD delete mode 100644 tensorflow-whl/tensorflow-gcs-config/third_party/tensorflow/BUILD.tpl delete mode 100644 tensorflow-whl/tensorflow-gcs-config/third_party/tensorflow/tf_configure.bzl diff --git a/README.md b/README.md index 1c8481bf..bd065b4a 100644 --- a/README.md +++ b/README.md @@ -74,17 +74,4 @@ docker run --runtime nvidia --rm -it kaggle/python-gpu-build /bin/bash docker run --runtime nvidia --rm -it gcr.io/kaggle-gpu-images/python /bin/bash ``` -To ensure your container can access the GPU, follow the instructions posted [here](https://github.com/Kaggle/docker-python/issues/361#issuecomment-448093930). - -## Tensorflow custom pre-built wheel - -A Tensorflow custom pre-built wheel is used mainly for: - -* Faster build time: Building tensorflow from sources takes ~1h. Keeping this process outside the main build allows faster iterations when working on our Dockerfiles. - -Building Tensorflow from sources: - -* Increase performance: When building from sources, we can leverage CPU specific optimizations -* Is required: Tensorflow with GPU support must be built from sources - -The [Dockerfile](tensorflow-whl/Dockerfile) and the [instructions](tensorflow-whl/README.md) can be found in the [tensorflow-whl folder/](tensorflow-whl/). +To ensure your container can access the GPU, follow the instructions posted [here](https://github.com/Kaggle/docker-python/issues/361#issuecomment-448093930). \ No newline at end of file diff --git a/tensorflow-whl/CHANGELOG.md b/tensorflow-whl/CHANGELOG.md deleted file mode 100644 index 1dfad51e..00000000 --- a/tensorflow-whl/CHANGELOG.md +++ /dev/null @@ -1,26 +0,0 @@ -* `1.11.0-py36`: TensorFlow 1.11.0 wheels built with python 3.6 -* `1.12.0-py36`: TensorFlow 1.12.0 wheels with CUDA 9.2 -* `1.13.1-py36`: TensorFlow 1.13.1 wheels with CUDA 10.0 -* `1.13.1-py36-2`: TensorFlow 1.13.1 wheels with CUDA 10.0 and bump anaconda3 base image version to 5.3.0 -* `1.13.1-py37`: TensorFlow 1.13.1 with Python 3.7.0 and bump anaconda3 base image version to 5.3.0 -* `1.13.1-py37-2`: TensorFlow 1.13.1 with Python 3.7.3 -* `1.14.0-py36`: TensorFlow 1.14.0 with Python 3.6 -* `2.0.0-rc1-py36`: TensorFlow 2.0.0 RC1 with Python 3.6 -* `2.0.0-py36`: TensorFlow 2.0.0 with Python 3.6 -* `2.1.0-rc0-py36`: TensorFlow 2.1.0-rc0 with Python 3.6 -* `2.1.0-rc2-py36`: TensorFlow 2.1.0-rc2 with Python 3.6 -* `2.1.0-py36`: TensorFlow 2.1.0 with Python 3.6 -* `2.1.0-py36-2`: TensorFlow 2.1.0 with CUDA 10.1 -* `2.1.0-py37`: TensorFlow 2.1.0 with Python 3.7 -* `2.1.0-py37-2`: TensorFlow 2.1.0 with Python 3.7 & DLVM base image -* `2.1.0-py37-3`: TensorFlow 2.1.0 with Python 3.7, DLVM base image, tensorflow-gcs-config -* `2.2.0-py37`: TensorFlow 2.2.0 with Python 3.7 -* `2.2.0-py37-2`: TensorFlow 2.2.0 with Python 3.7 & TFA -* `2.3.0-py37`: TensorFlow 2.3.0 with Python 3.7 -* `2.3.1-py37`: TensorFlow 2.3.1 with Python 3.7 -* `2.3.1-py37-2`: TensorFlow 2.3.1 & TFA 0.11 with Python 3.7 -* `2.4.0-py37`: TensorFlow 2.4.0 & Python 3.7 & TFA 0.12 -* `2.4.0-py37-2`: TensorFlow 2.4.0 & m61 base image -* `2.4.0-py37-3`: TensorFlow 2.4.0 & 7.0 CUDA compute capability -* `2.4.1-py37`: TensorFlow 2.4.1 & TFA 0.12.1 -* `2.4.1-py37-2`: TensorFlow 2.4.1 & CUDA 11.0 \ No newline at end of file diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile deleted file mode 100644 index 0cc4140a..00000000 --- a/tensorflow-whl/Dockerfile +++ /dev/null @@ -1,131 +0,0 @@ -FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 AS nvidia -FROM gcr.io/deeplearning-platform-release/base-cpu:m73 - -# Avoid interactive configuration prompts/dialogs during apt-get. -ENV DEBIAN_FRONTEND=noninteractive - -# This is necessary to for apt to access HTTPS sources -RUN apt-get update && \ - apt-get install apt-transport-https - -# Cuda support -COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ -COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ -COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg -# See b/142337634#comment28 -RUN sed -i 's/deb https:\/\/developer.download.nvidia.com/deb http:\/\/developer.download.nvidia.com/' /etc/apt/sources.list.d/*.list - -# Ensure the cuda libraries are compatible with the GPU image. -# TODO(b/120050292): Use templating to keep in sync. -ENV CUDA_MAJOR_VERSION=11 -ENV CUDA_MINOR_VERSION=0 -ENV CUDA_VERSION=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION -LABEL com.nvidia.volumes.needed="nvidia_driver" -LABEL com.nvidia.cuda.version="${CUDA_VERSION}" -ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -# The stub is useful to us both for built-time linking and run-time linking, on CPU-only systems. -# When intended to be used with actual GPUs, make sure to (besides providing access to the host -# CUDA user libraries, either manually or through the use of nvidia-docker) exclude them. One -# convenient way to do so is to obscure its contents by a bind mount: -# docker run .... -v /non-existing-directory:/usr/local/cuda/lib64/stubs:ro ... -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH" -ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility -ENV NVIDIA_REQUIRE_CUDA="cuda>=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION" -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-cupti-$CUDA_VERSION \ - cuda-cudart-$CUDA_VERSION \ - cuda-cudart-dev-$CUDA_VERSION \ - cuda-libraries-$CUDA_VERSION \ - cuda-libraries-dev-$CUDA_VERSION \ - cuda-nvml-dev-$CUDA_VERSION \ - cuda-minimal-build-$CUDA_VERSION \ - cuda-command-line-tools-$CUDA_VERSION \ - libcudnn8=8.0.4.30-1+cuda$CUDA_VERSION \ - libcudnn8-dev=8.0.4.30-1+cuda$CUDA_VERSION \ - libnccl2=2.7.8-1+cuda$CUDA_VERSION \ - libnccl-dev=2.7.8-1+cuda$CUDA_VERSION && \ - ln -s /usr/local/cuda-$CUDA_VERSION /usr/local/cuda && \ - ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 - -# Use Bazelisk to ensure the proper bazel version is used. -RUN cd /usr/local/src && \ - wget --no-verbose "https://github.com/bazelbuild/bazelisk/releases/download/v1.7.4/bazelisk-linux-amd64" && \ - mv bazelisk-linux-amd64 /usr/local/bin/bazel && \ - chmod u+x /usr/local/bin/bazel - -# Fetch TensorFlow & install dependencies. -RUN cd /usr/local/src && \ - git clone https://github.com/tensorflow/tensorflow && \ - cd tensorflow && \ - git checkout tags/v2.4.1 && \ - pip install keras_applications --no-deps && \ - pip install keras_preprocessing --no-deps - -# Create a TensorFlow wheel for CPU -RUN cd /usr/local/src/tensorflow && \ - cat /dev/null | ./configure && \ - bazel build --config=opt \ - --config=v2 \ - --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ - //tensorflow/tools/pip_package:build_pip_package && \ - bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_cpu && \ - bazel clean - -# Install TensorFlow CPU wheel which is required to build the GCS & addons libraries. -RUN pip install /tmp/tensorflow_cpu/tensorflow*.whl - -# Build TensorFlow addons library against TensorFlow CPU. -RUN cd /usr/local/src/ && \ - git clone https://github.com/tensorflow/addons && \ - cd addons && \ - git checkout tags/v0.12.1 && \ - python ./configure.py && \ - bazel build --enable_runfiles build_pip_pkg && \ - bazel-bin/build_pip_pkg /tmp/tfa_cpu && \ - bazel clean - -# Build tensorflow_gcs_config library against TensorFlow CPU. -ADD tensorflow-gcs-config /usr/local/src/tensorflow_gcs_config/ -RUN cd /usr/local/src/tensorflow_gcs_config && \ - apt-get install -y libcurl4-openssl-dev && \ - python setup.py bdist_wheel -d /tmp/tensorflow_gcs_config && \ - bazel clean - -# Create a tensorflow wheel for GPU/cuda -ENV TF_NEED_CUDA=1 -ENV TF_CUDA_VERSION=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION -# 3.7 (K80), 6.0 (P100), 7.0 (V100), 7.5 (T4): https://developer.nvidia.com/cuda-gpus -ENV TF_CUDA_COMPUTE_CAPABILITIES=3.7,6.0,7.0,7.5 -ENV TF_CUDNN_VERSION=8 -ENV TF_NCCL_VERSION=2 -ENV NCCL_INSTALL_PATH=/usr/ - -RUN cd /usr/local/src/tensorflow && \ - # TF_NCCL_INSTALL_PATH is used for both libnccl.so.2 and libnccl.h. Make sure they are both accessible from the same directory. - ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/lib/ && \ - cat /dev/null | ./configure && \ - echo "/usr/local/cuda-${TF_CUDA_VERSION}/targets/x86_64-linux/lib/stubs" > /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \ - bazel build --config=opt \ - --config=v2 \ - --config=cuda \ - --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ - //tensorflow/tools/pip_package:build_pip_package && \ - rm /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \ - bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_gpu && \ - bazel clean - -# Install TensorFlow GPU wheel which to build addons against. -RUN pip install /tmp/tensorflow_gpu/tensorflow*.whl - -# Build TensorFlow addons library against TensorFlow GPU. -ENV CUDA_TOOLKIT_PATH=/usr/local/cuda -ENV CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu -RUN cd /usr/local/src/addons && \ - python ./configure.py && \ - bazel build --enable_runfiles build_pip_pkg && \ - bazel-bin/build_pip_pkg /tmp/tfa_gpu && \ - bazel clean - -# Print out the built .whl files -RUN ls -R /tmp/tensorflow* diff --git a/tensorflow-whl/Jenkinsfile b/tensorflow-whl/Jenkinsfile deleted file mode 100644 index 9a9cbd60..00000000 --- a/tensorflow-whl/Jenkinsfile +++ /dev/null @@ -1,36 +0,0 @@ -pipeline { - agent { label 'ephemeral-linux-gpu' } - options { - // The Build GPU stage depends on the image from the Push CPU stage - disableConcurrentBuilds() - } - environment { - GIT_COMMIT_SHORT = sh(returnStdout: true, script:"git rev-parse --short=7 HEAD").trim() - GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() - GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() - GIT_COMMIT_SUMMARY = "`` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" - } - - stages { - stage('Build') { - steps { - sh '''#!/bin/bash - set -exo pipefail - - cd tensorflow-whl/ - ./build | ts - ''' - } - } - stage('Push') { - steps { - sh '''#!/bin/bash - set -exo pipefail - - cd tensorflow-whl/ - ./push ${GIT_BRANCH}-staging - ''' - } - } - } -} diff --git a/tensorflow-whl/README.md b/tensorflow-whl/README.md deleted file mode 100644 index 02c74d14..00000000 --- a/tensorflow-whl/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Build new Tensorflow wheels - -``` -./build -``` - -# Push the new wheels (Kaggle Engineers only) - -1. Add an entry in the [CHANGELOG](CHANGELOG.md) with an appropriate `LABEL`. -2. Push the new image using the `LABEL` you picked above. - - ``` - ./push LABEL - ``` - -# Use the new wheels - -Update the line below in the [CPU Dockerfile](../Dockerfile) and the [GPU Dockerfile](../gpu.Dockerfile) to use the new `LABEL`. - -To use wheels built locally: -``` -FROM kaggle/python-tensorflow-whl as tensorflow_whl -``` - -To use our pre-built wheels: -``` -FROM gcr.io/kaggle-images/python-tensorflow-whl: