Train_Colab.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 457,
          "referenced_widgets": [
            "dba106bbaf184380bdb32b0409dad1db",
            "c142ece118784b8d9c277bb9037be7ab",
            "66ec6b636f1f4900b18192b11dca250e",
            "8e50c8dad02046a59b46d5cf6b7d5779",
            "e3cc3be83c5944589ada73ef002c930e",
            "97d49e68226d47669e5868e3d766760a",
            "5f7825c7242949ff8d545d7ca715db4e",
            "793dc8bf53d543e7961d4ccd0222effa"
          ]
        },
        "id": "P3bFvrXa-PgC",
        "outputId": "127e3027-8a84-4d50-ed5c-2dbbfdd7da80"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "check environment \n",
            "I'm running on Colab\n",
            "Cuda available True\n",
            "installing required packages ...\n",
            "Manage dataset\n",
            "Mounted at /content/drive\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Tracking run with wandb version 0.15.7"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Run data is saved locally in <code>/content/wandb/run-20230726_175741-daz52r89</code>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Syncing run <strong><a href='https://wandb.ai/team-invonto/Card_detection/runs/daz52r89' target=\"_blank\">stellar-morning-19</a></strong> to <a href='https://wandb.ai/team-invonto/Card_detection' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View project at <a href='https://wandb.ai/team-invonto/Card_detection' target=\"_blank\">https://wandb.ai/team-invonto/Card_detection</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View run at <a href='https://wandb.ai/team-invonto/Card_detection/runs/daz52r89' target=\"_blank\">https://wandb.ai/team-invonto/Card_detection/runs/daz52r89</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dowload data from the wandb to temp\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact Dataset_July2023_train213_val31:v0, 73.86MB. 2 files... \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   2 of 2 files downloaded.  \n",
            "Done. 0:0:1.5\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(Label(value='0.001 MB of 0.001 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dba106bbaf184380bdb32b0409dad1db"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View run <strong style=\"color:#cdcd00\">stellar-morning-19</strong> at: <a href='https://wandb.ai/team-invonto/Card_detection/runs/daz52r89' target=\"_blank\">https://wandb.ai/team-invonto/Card_detection/runs/daz52r89</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Find logs at: <code>./wandb/run-20230726_175741-daz52r89/logs</code>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "import os\n",
        "import subprocess\n",
        "import sys\n",
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import cv2\n",
        "import zipfile\n",
        "from torchvision import models\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import json\n",
        "\n",
        "\n",
        "##############################################################################\n",
        "print(\"check environment \")\n",
        "##############################################################################\n",
        "\n",
        "if 'COLAB_GPU' in os.environ:\n",
        "   print(\"I'm running on Colab\")\n",
        "\n",
        "   def install(package):\n",
        "       subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package])\n",
        "\n",
        "   print(\"Cuda available\",torch.cuda.is_available())\n",
        "   print(\"installing required packages ...\")\n",
        "   install(\"segmentation-models-pytorch\")\n",
        "   install(\"lightning\")\n",
        "   install(\"wandb\")\n",
        "   install(\"onnx\")\n",
        "   install(\"albumentations\")\n",
        "\n",
        "\n",
        "import segmentation_models_pytorch as smp\n",
        "import albumentations as A\n",
        "import albumentations.augmentations.functional as F\n",
        "from albumentations.pytorch import ToTensorV2\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "import pytorch_lightning as pl\n",
        "from lightning.pytorch.utilities.model_summary import ModelSummary\n",
        "from pytorch_lightning.loggers import WandbLogger\n",
        "\n",
        "import wandb\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "##############################################################################\n",
        "print(\"Manage dataset\")\n",
        "##############################################################################\n",
        "\n",
        "if 'COLAB_GPU' in os.environ:\n",
        "\n",
        "   from google.colab import drive\n",
        "   drive.mount('/content/drive')\n",
        "\n",
        "\n",
        "   def load_data_to_tmp(folder_name):\n",
        "    zip_ref = zipfile.ZipFile('/tmp/'+folder_name, 'r') #Opens the zip file in read mode\n",
        "    zip_ref.extractall('/tmp') #Extracts the files into the /tmp folder\n",
        "    zip_ref.close()\n",
        "\n",
        "   run=wandb.init(project=\"Card_detection\")\n",
        "   print(\"dowload data from the wandb to temp\")\n",
        "   artifact = run.use_artifact('team-invonto/Card_detection/Dataset_July2023_train213_val31:v0', type='compressed_480x320')\n",
        "   artifact.download('/tmp/')\n",
        "   load_data_to_tmp('Train.zip')\n",
        "   load_data_to_tmp('Validation.zip')\n",
        "   data_path='/tmp/'\n",
        "   wandb.finish()\n",
        "\n",
        "\n",
        "else:\n",
        "   data_path='/home/tatyana/Work_Invonto/Dataset/Card_Dataset/Dataset_ready_July2023'\n",
        "   print('running local')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "Vy21psk59sLq"
      },
      "outputs": [],
      "source": [
        "############################################################################################################33\n",
        "#### Card Dataset\n",
        "\n",
        "class CardDataset(Dataset):\n",
        "    \"\"\"NailDataset dataset.\n",
        "    img generator that take file names and selected poligons from csv_file\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, csv_file, root_dir, transform=None, augmentation=None):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            csv_file (string): Path to the csv file with annotations.\n",
        "            root_dir (string): Directory with all the images.\n",
        "            transform (callable, optional): Optional transform to be applied\n",
        "                on a sample.\n",
        "\n",
        "            nail_dataset[img_number_in_csv][1] -image\n",
        "            nail_dataset[img_number_in_csv][0] -corresponding mask image\n",
        "        \"\"\"\n",
        "        self.nail_frame = pd.read_csv(csv_file)\n",
        "        self.root_dir = root_dir\n",
        "        self.transform = transform\n",
        "        self.augmentation = augmentation\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.nail_frame.shape[0]\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        file_id=self.nail_frame.iloc[idx][\"filename\"]\n",
        "        #\"read image\"\n",
        "        img_name = os.path.join(self.root_dir,self.nail_frame.iloc[idx][\"filename\"])\n",
        "        img_name=img_name.split('.')[0]+'.png'\n",
        "        image = cv2.imread(img_name)\n",
        "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
        "\n",
        "        # read mask\n",
        "        mask_name=os.path.join(self.root_dir,self.nail_frame.iloc[idx][\"mask\"])\n",
        "        mask = cv2.imread(mask_name,cv2.IMREAD_UNCHANGED)\n",
        "\n",
        "        # required for albumentation\n",
        "        mask = mask.astype(np.float32)\n",
        "        mask[mask == 0] = 0.0\n",
        "        mask[mask == 255] = 1.0\n",
        "\n",
        "        if self.transform:\n",
        "            sample = self.transform(image=image, mask=mask)\n",
        "            image = sample['image']\n",
        "            mask=sample['mask']\n",
        "            mask=mask.unsqueeze(0)\n",
        "\n",
        "        return image,mask\n",
        "\n",
        "train_transform=A.Compose([\n",
        "        A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=30, p=0.5,border_mode=cv2.BORDER_CONSTANT),\n",
        "        A.RGBShift(r_shift_limit=25, g_shift_limit=25, b_shift_limit=25, p=0.5),\n",
        "        A.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.5),\n",
        "        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n",
        "        ToTensorV2()])\n",
        "\n",
        "\n",
        "validation_transform =A.Compose([\n",
        "        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n",
        "        ToTensorV2()])\n",
        "\n",
        "\n",
        "############################################################################################################33\n",
        "#### Card Model\n",
        "\n",
        "class CardModel(pl.LightningModule):\n",
        "\n",
        "    def __init__(self,arch,encoder_name,encoder_weights,learning_rate,name_optimizer,frozen_encoder=True):\n",
        "        super().__init__()\n",
        "        self.lr=learning_rate\n",
        "        self.optimizer_name=name_optimizer\n",
        "        self.save_hyperparameters()\n",
        "        self.training_step_outputs = []\n",
        "        self.validation_step_outputs = []\n",
        "\n",
        "        self.model = smp.create_model(\n",
        "            arch,\n",
        "            encoder_name=encoder_name,\n",
        "            encoder_weights=encoder_weights,\n",
        "            classes=1,\n",
        "            #activation='sigmoid'\n",
        "            )\n",
        "\n",
        "        # for image segmentation dice loss could be the best first choice\n",
        "        self.loss_fn = smp.losses.DiceLoss(smp.losses.BINARY_MODE,from_logits=True)\n",
        "\n",
        "        if frozen_encoder==True:\n",
        "            for child in self.model.encoder.children():\n",
        "                for param in child.parameters():\n",
        "                    param.requires_grad = False\n",
        "\n",
        "\n",
        "    def forward(self, image):\n",
        "        mask = self.model(image)\n",
        "        return mask\n",
        "\n",
        "    def training_step(self, batch, batch_idx):\n",
        "        # training_step defines the train loop.\n",
        "        # it is independent of forward\n",
        "\n",
        "        image=batch[0]\n",
        "        # assert image (batch_size,num_channels,height,width)\n",
        "        assert image.ndim == 4\n",
        "        # assert the image dim is suitable for Unet\n",
        "        h, w = image.shape[2:]\n",
        "        assert h % 32 == 0 and w % 32 == 0\n",
        "\n",
        "        mask=batch[1]\n",
        "        assert mask.ndim == 4\n",
        "        assert mask.max()<=1 and  mask.min()>=0 # check that the mask between [0,1] not in range [0,255]\n",
        "\n",
        "        # calculate loss\n",
        "        output_mask = self.forward(image)\n",
        "        loss=self.loss_fn(output_mask,mask)\n",
        "\n",
        "        pred_mask=(output_mask.sigmoid() >= 0.5).float()\n",
        "\n",
        "\n",
        "\n",
        "        # calculate logs\n",
        "        tp, fp, fn, tn = smp.metrics.get_stats(pred_mask.long(), mask.long(), mode=\"binary\")\n",
        "\n",
        "        train_batch_dictionary={\"tp\":tp,\n",
        "                          \"fp\":fp,\n",
        "                          \"fn\":fn,\n",
        "                          \"tn\":tn}\n",
        "\n",
        "        self.training_step_outputs.append(train_batch_dictionary)\n",
        "        # logs metrics for each training_step,\n",
        "        # and the average across the epoch, to the progress bar and logger\n",
        "        self.log(\"train_loss\", loss, on_epoch=True, prog_bar=True, logger=True)\n",
        "\n",
        "        return loss\n",
        "\n",
        "    def on_train_epoch_end(self):\n",
        "        # aggregate step metics\n",
        "        outputs=self.training_step_outputs\n",
        "        tp = torch.cat([x[\"tp\"] for x in outputs])\n",
        "        fp = torch.cat([x[\"fp\"] for x in outputs])\n",
        "        fn = torch.cat([x[\"fn\"] for x in outputs])\n",
        "        tn = torch.cat([x[\"tn\"] for x in outputs])\n",
        "\n",
        "\n",
        "        per_image_iou = smp.metrics.iou_score(tp, fp, fn, tn, reduction=\"micro-imagewise\") # mean (per image)\n",
        "        dataset_iou = smp.metrics.iou_score(tp, fp, fn, tn, reduction=\"micro\") # sum tp,fp,fn,tn over all dataset\n",
        "\n",
        "        metrics = {\n",
        "            \"train_per_image_iou\": per_image_iou,\n",
        "        #    \"train_dataset_iou\": dataset_iou,\n",
        "        }\n",
        "\n",
        "        self.log_dict(metrics, prog_bar=True)\n",
        "        self.training_step_outputs.clear()\n",
        "\n",
        "\n",
        "\n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        # training_step defines the train loop.\n",
        "        # it is independent of forward\n",
        "\n",
        "        image=batch[0]\n",
        "        # assert image (batch_size,num_channels,height,width)\n",
        "        assert image.ndim == 4\n",
        "        # assert the image dim is suitable for Unet\n",
        "        h, w = image.shape[2:]\n",
        "        assert h % 32 == 0 and w % 32 == 0\n",
        "\n",
        "        mask=batch[1]\n",
        "        assert mask.ndim == 4\n",
        "        assert mask.max()<=1 and  mask.min()>=0 # check that the mask between [0,1] not in range [0,255]\n",
        "\n",
        "        # calculate loss\n",
        "        output_mask = self.forward(image)\n",
        "        loss=self.loss_fn(output_mask,mask)\n",
        "\n",
        "        pred_mask=(output_mask.sigmoid() >= 0.5).float()\n",
        "\n",
        "\n",
        "        # calculate logs\n",
        "        tp, fp, fn, tn = smp.metrics.get_stats(pred_mask.long(), mask.long(), mode=\"binary\")\n",
        "\n",
        "        validation_batch_dictionary={\"loss\":loss,\n",
        "                          \"tp\":tp,\n",
        "                          \"fp\":fp,\n",
        "                          \"fn\":fn,\n",
        "                          \"tn\":tn}\n",
        "\n",
        "        self.validation_step_outputs.append(validation_batch_dictionary)\n",
        "        # logs metrics for each training_step,\n",
        "        # and the average across the epoch, to the progress bar and logger\n",
        "        self.log(\"valid_loss\", loss, on_epoch=True, prog_bar=True, logger=True)\n",
        "\n",
        "        return loss\n",
        "\n",
        "\n",
        "    def on_validation_epoch_end(self):\n",
        "        # aggregate step metics\n",
        "        outputs= self.validation_step_outputs\n",
        "        tp = torch.cat([x[\"tp\"] for x in outputs])\n",
        "        fp = torch.cat([x[\"fp\"] for x in outputs])\n",
        "        fn = torch.cat([x[\"fn\"] for x in outputs])\n",
        "        tn = torch.cat([x[\"tn\"] for x in outputs])\n",
        "\n",
        "\n",
        "        per_image_iou = smp.metrics.iou_score(tp, fp, fn, tn, reduction=\"micro-imagewise\") # mean (per image)\n",
        "        #dataset_iou = smp.metrics.iou_score(tp, fp, fn, tn, reduction=\"micro\") # sum tp,fp,fn,tn over all dataset\n",
        "\n",
        "        metrics = {\n",
        "            \"validation_per_image_iou\": per_image_iou,\n",
        "            #\"validation_dataset_iou\": dataset_iou,\n",
        "        }\n",
        "\n",
        "        self.log_dict(metrics, prog_bar=True)\n",
        "        self.validation_step_outputs.clear()\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        if  self.optimizer_name=='Adam':\n",
        "            return torch.optim.Adam(self.parameters(), lr=(self.lr or self.learning_rate))\n",
        "        if  self.optimizer_name==\"Sgd\":\n",
        "            return torch.optim.SGD(self.parameters(), lr=(self.lr or self.learning_rate), momentum=0.9)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "wW_DV87i9sLt"
      },
      "outputs": [],
      "source": [
        "def train_run(config=None):\n",
        "\n",
        "    N_EPOCH=20\n",
        "    ENCODER = 'resnet18'\n",
        "    Frozen_Encoder_Flag=True\n",
        "\n",
        "    # Get hyperparameters from the run configs\n",
        "    wandb.init(config=config)\n",
        "    config = wandb.config\n",
        "\n",
        "\n",
        "\n",
        "    #data_path='/home/tatyana/Work_Invonto/Dataset/Card_Dataset/Dataset_ready_July2023'\n",
        "    data_path='/tmp/'\n",
        "\n",
        "    ### Load Data\n",
        "    n_cpu = os.cpu_count()\n",
        "    #train\n",
        "    x_train_path=os.path.join(data_path,'Train')\n",
        "    #y_train_dataloader=os.path.join(x_train_path,'Train_DataLoader.csv')\n",
        "    y_train_dataloader=os.path.join(x_train_path,'Train_DataLoader_sample96.csv')\n",
        "    train_dataset = CardDataset(y_train_dataloader, x_train_path,transform=train_transform, augmentation=None)\n",
        "    train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True,drop_last=True,num_workers=n_cpu)\n",
        "    #print('Number of training samples:', len(train_dataset))\n",
        "\n",
        "    # validation\n",
        "    x_valid_path = os.path.join(data_path, 'Validation')\n",
        "    y_valid_dataloader = os.path.join(x_valid_path, 'Validation_DataLoader.csv')\n",
        "    valid_dataset = CardDataset(y_valid_dataloader,x_valid_path,transform=validation_transform, augmentation=None)\n",
        "    valid_loader = DataLoader(valid_dataset, batch_size=config.batch_size, shuffle=False,num_workers=n_cpu)\n",
        "    #print('Number of validation samples:', len(valid_dataset))\n",
        "\n",
        "\n",
        "    ### Define Model\n",
        "    card_model = CardModel(\"Unet\", ENCODER ,\"imagenet\",config.learning_rate,config.optimizer,frozen_encoder=Frozen_Encoder_Flag)\n",
        "\n",
        "    wandb_logger = WandbLogger(project=\"Card_detection\")\n",
        "    trainer = pl.Trainer(max_epochs=N_EPOCH,\n",
        "                     logger=wandb_logger,\n",
        "                     log_every_n_steps=1,\n",
        "                     default_root_dir=\"/content/drive/MyDrive/Dataset/\")\n",
        "    trainer.fit(\n",
        "        model=card_model,\n",
        "        train_dataloaders=train_loader,\n",
        "        val_dataloaders=valid_loader)\n",
        "\n",
        "    name=\"CardModel_\"+run.id\n",
        "    path = \"/content/drive/MyDrive/Dataset/\"\n",
        "    model_name=path+name+'.onnx'\n",
        "    torch.onnx.export(card_model,\n",
        "                  torch.randn(1,3,480,320).to('cpu'),\n",
        "                  model_name,\n",
        "                  input_names = ['input'],              # the model's input names\n",
        "                  output_names = ['output'])\n",
        "    wandb.save(model_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "qYr94Fth9sLu",
        "outputId": "3689892f-0bcf-4053-fbd0-74b81270a3e7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error in callback <function _WandbInit._resume_backend at 0x7c91d3fcdd80> (for pre_run_cell):\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "BrokenPipeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mBrokenPipeError\u001b[0m                           Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/backcall/backcall.py\u001b[0m in \u001b[0;36madapted\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;31m#            print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0madapted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\u001b[0m in \u001b[0;36m_resume_backend\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    422\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"resuming backend\"\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 424\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpublish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    426\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_jupyter_teardown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface.py\u001b[0m in \u001b[0;36mpublish_resume\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    737\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpublish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m         \u001b[0mresume\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mResumeRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 739\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    740\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    741\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\u001b[0m in \u001b[0;36m_publish_resume\u001b[0;34m(self, resume)\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresume\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mResumeRequest\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    352\u001b[0m         \u001b[0mrec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    354\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRunRecord\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\u001b[0m in \u001b[0;36m_publish\u001b[0;34m(self, record, local)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pb.Record\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_assign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_record_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     def _communicate_async(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_record_publish\u001b[0;34m(self, record)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0mserver_req\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mServerRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mserver_req\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_publish\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCopyFrom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mserver_req\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_extract_packet_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbytes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_server_request\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_send_message\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"<BI\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"W\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sendall_with_error_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheader\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_sendall_with_error_handle\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonotonic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m                 \u001b[0msent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m                 \u001b[0;31m# sent equal to 0 indicates a closed socket\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mBrokenPipeError\u001b[0m: [Errno 32] Broken pipe"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.login() after wandb.init() has no effect.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Create sweep with ID: xmeyeqxe\n",
            "Sweep URL: https://wandb.ai/team-invonto/Card_detection/sweeps/xmeyeqxe\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: 40v9ix71 with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlearning_rate: 0.07690522745751438\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \toptimizer: Sgd\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 1143, in init\n",
            "    wi.setup(kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 208, in setup\n",
            "    with telemetry.context(obj=self._init_telemetry_obj) as tel:\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\", line 42, in __exit__\n",
            "    self._run._telemetry_callback(self._obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 713, in _telemetry_callback\n",
            "    self._telemetry_flush()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 724, in _telemetry_flush\n",
            "    self._backend.interface._publish_telemetry(self._telemetry_obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 101, in _publish_telemetry\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n",
            "Exception in thread Thread-113 (_run_job):\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 1143, in init\n",
            "    wi.setup(kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 208, in setup\n",
            "    with telemetry.context(obj=self._init_telemetry_obj) as tel:\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\", line 42, in __exit__\n",
            "    self._run._telemetry_callback(self._obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 713, in _telemetry_callback\n",
            "    self._telemetry_flush()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 724, in _telemetry_flush\n",
            "    self._backend.interface._publish_telemetry(self._telemetry_obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 101, in _publish_telemetry\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n",
            "\n",
            "The above exception was the direct cause of the following exception:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/agents/pyagent.py\", line 298, in _run_job\n",
            "    self._function()\n",
            "  File \"<ipython-input-26-137f3730e8fd>\", line 8, in train_run\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 1185, in init\n",
            "    raise Error(\"An unexpected error occurred\") from error_seen\n",
            "wandb.errors.Error: An unexpected error occurred\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 1016, in _bootstrap_inner\n",
            "    self.run()\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 953, in run\n",
            "    self._target(*self._args, **self._kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/agents/pyagent.py\", line 303, in _run_job\n",
            "    wandb.finish(exit_code=1)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 3711, in finish\n",
            "    wandb.run.finish(exit_code=exit_code, quiet=quiet)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 390, in wrapper\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 331, in wrapper\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1879, in finish\n",
            "    return self._finish(exit_code, quiet)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1886, in _finish\n",
            "    with telemetry.context(run=self) as tel:\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\", line 42, in __exit__\n",
            "    self._run._telemetry_callback(self._obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 713, in _telemetry_callback\n",
            "    self._telemetry_flush()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 724, in _telemetry_flush\n",
            "    self._backend.interface._publish_telemetry(self._telemetry_obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 101, in _publish_telemetry\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: 1lfen6o2 with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 64\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlearning_rate: 0.06846743979846928\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \toptimizer: Sgd\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 1143, in init\n",
            "    wi.setup(kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 208, in setup\n",
            "    with telemetry.context(obj=self._init_telemetry_obj) as tel:\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\", line 42, in __exit__\n",
            "    self._run._telemetry_callback(self._obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 713, in _telemetry_callback\n",
            "    self._telemetry_flush()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 724, in _telemetry_flush\n",
            "    self._backend.interface._publish_telemetry(self._telemetry_obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 101, in _publish_telemetry\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n",
            "Exception in thread Thread-114 (_run_job):\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 1143, in init\n",
            "    wi.setup(kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 208, in setup\n",
            "    with telemetry.context(obj=self._init_telemetry_obj) as tel:\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\", line 42, in __exit__\n",
            "    self._run._telemetry_callback(self._obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 713, in _telemetry_callback\n",
            "    self._telemetry_flush()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 724, in _telemetry_flush\n",
            "    self._backend.interface._publish_telemetry(self._telemetry_obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 101, in _publish_telemetry\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n",
            "\n",
            "The above exception was the direct cause of the following exception:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/agents/pyagent.py\", line 298, in _run_job\n",
            "    self._function()\n",
            "  File \"<ipython-input-26-137f3730e8fd>\", line 8, in train_run\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\", line 1185, in init\n",
            "    raise Error(\"An unexpected error occurred\") from error_seen\n",
            "wandb.errors.Error: An unexpected error occurred\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 1016, in _bootstrap_inner\n",
            "    self.run()\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 953, in run\n",
            "    self._target(*self._args, **self._kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/agents/pyagent.py\", line 303, in _run_job\n",
            "    wandb.finish(exit_code=1)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 3711, in finish\n",
            "    wandb.run.finish(exit_code=exit_code, quiet=quiet)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 390, in wrapper\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 331, in wrapper\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1879, in finish\n",
            "    return self._finish(exit_code, quiet)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1886, in _finish\n",
            "    with telemetry.context(run=self) as tel:\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\", line 42, in __exit__\n",
            "    self._run._telemetry_callback(self._obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 713, in _telemetry_callback\n",
            "    self._telemetry_flush()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 724, in _telemetry_flush\n",
            "    self._backend.interface._publish_telemetry(self._telemetry_obj)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 101, in _publish_telemetry\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Ctrl + C detected. Stopping sweep.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error in callback <function _WandbInit._pause_backend at 0x7c91d42c0e50> (for post_run_cell):\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "BrokenPipeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mBrokenPipeError\u001b[0m                           Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/backcall/backcall.py\u001b[0m in \u001b[0;36madapted\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;31m#            print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0madapted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\u001b[0m in \u001b[0;36m_pause_backend\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    417\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    418\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pausing backend\"\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 419\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpublish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    421\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_resume_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface.py\u001b[0m in \u001b[0;36mpublish_pause\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    729\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpublish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m         \u001b[0mpause\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPauseRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 731\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpause\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    732\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\u001b[0m in \u001b[0;36m_publish_pause\u001b[0;34m(self, pause)\u001b[0m\n\u001b[1;32m    347\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpause\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPauseRequest\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m         \u001b[0mrec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpause\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpause\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresume\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mResumeRequest\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\u001b[0m in \u001b[0;36m_publish\u001b[0;34m(self, record, local)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pb.Record\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_assign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_record_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     def _communicate_async(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_record_publish\u001b[0;34m(self, record)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0mserver_req\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mServerRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mserver_req\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_publish\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCopyFrom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mserver_req\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_extract_packet_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbytes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_server_request\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_send_message\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"<BI\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"W\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sendall_with_error_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheader\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_sendall_with_error_handle\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonotonic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m                 \u001b[0msent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m                 \u001b[0;31m# sent equal to 0 indicates a closed socket\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mBrokenPipeError\u001b[0m: [Errno 32] Broken pipe"
          ]
        }
      ],
      "source": [
        "sweep_config = {\n",
        "    \"method\": \"bayes\",\n",
        "    \"metric\":{\"name\":\"validation_per_image_iou\",\"goal\":\"maximize\"},\n",
        "    \"parameters\":{\n",
        "        \"optimizer\":{'values':['Adam','Sgd']},\n",
        "        'batch_size': {'values': [16,32,64]},\n",
        "        \"learning_rate\": {\"distribution\": \"uniform\", \"min\": 1e-5, \"max\": 0.1}\n",
        "        }\n",
        "    }\n",
        "\n",
        "sweep_id = wandb.sweep(sweep_config, project=\"Card_detection\")\n",
        "wandb.agent(sweep_id, train_run, count=30)\n",
        "wandb.finish()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "Q3_rJP7fAflx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3e301f64-6878-47c4-b568-276458fd8ad3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error in callback <function _WandbInit._resume_backend at 0x7c91d3fcdd80> (for pre_run_cell):\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "BrokenPipeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mBrokenPipeError\u001b[0m                           Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/backcall/backcall.py\u001b[0m in \u001b[0;36madapted\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;31m#            print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0madapted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\u001b[0m in \u001b[0;36m_resume_backend\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    422\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"resuming backend\"\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 424\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpublish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    426\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_jupyter_teardown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface.py\u001b[0m in \u001b[0;36mpublish_resume\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    737\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpublish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m         \u001b[0mresume\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mResumeRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 739\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    740\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    741\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\u001b[0m in \u001b[0;36m_publish_resume\u001b[0;34m(self, resume)\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresume\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mResumeRequest\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    352\u001b[0m         \u001b[0mrec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    354\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRunRecord\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\u001b[0m in \u001b[0;36m_publish\u001b[0;34m(self, record, local)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pb.Record\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_assign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_record_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     def _communicate_async(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_record_publish\u001b[0;34m(self, record)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0mserver_req\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mServerRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mserver_req\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_publish\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCopyFrom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mserver_req\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_extract_packet_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbytes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_server_request\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_send_message\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"<BI\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"W\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sendall_with_error_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheader\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_sendall_with_error_handle\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonotonic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m                 \u001b[0msent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m                 \u001b[0;31m# sent equal to 0 indicates a closed socket\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mBrokenPipeError\u001b[0m: [Errno 32] Broken pipe"
          ]
        },
        {
          "output_type": "error",
          "ename": "BrokenPipeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mBrokenPipeError\u001b[0m                           Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-a2bce46dc485>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mwandb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36mfinish\u001b[0;34m(exit_code, quiet)\u001b[0m\n\u001b[1;32m   3709\u001b[0m     \"\"\"\n\u001b[1;32m   3710\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mwandb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3711\u001b[0;31m         \u001b[0mwandb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexit_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mexit_code\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquiet\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquiet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    388\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDummy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 390\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    329\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m                 \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_attaching\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36mfinish\u001b[0;34m(self, exit_code, quiet)\u001b[0m\n\u001b[1;32m   1877\u001b[0m             \u001b[0mquiet\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSet\u001b[0m \u001b[0mto\u001b[0m \u001b[0mtrue\u001b[0m \u001b[0mto\u001b[0m \u001b[0mminimize\u001b[0m \u001b[0mlog\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1878\u001b[0m         \"\"\"\n\u001b[0;32m-> 1879\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_finish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexit_code\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquiet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1880\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1881\u001b[0m     def _finish(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36m_finish\u001b[0;34m(self, exit_code, quiet)\u001b[0m\n\u001b[1;32m   1884\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mquiet\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1885\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_quiet\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquiet\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1886\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mtelemetry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1887\u001b[0m             \u001b[0mtel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinish\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1888\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"finishing run {self._get_path()}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/telemetry.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, exctype, excinst, exctb)\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36m_telemetry_callback\u001b[0;34m(self, telem_obj)\u001b[0m\n\u001b[1;32m    711\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMergeFrom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtelem_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_obj_dirty\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 713\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_flush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    715\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_telemetry_flush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\u001b[0m in \u001b[0;36m_telemetry_flush\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    722\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mserialized\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_obj_flushed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    723\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 724\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish_telemetry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    725\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_obj_flushed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mserialized\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_telemetry_obj_dirty\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\u001b[0m in \u001b[0;36m_publish_telemetry\u001b[0;34m(self, telem)\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_telemetry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtelem\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTelemetryRecord\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0mrec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_record\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtelemetry\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtelem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_stats\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstats_dict\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStatsRecord\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\u001b[0m in \u001b[0;36m_publish\u001b[0;34m(self, record, local)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pb.Record\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_assign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_record_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     def _communicate_async(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_record_publish\u001b[0;34m(self, record)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0mserver_req\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mServerRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mserver_req\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_publish\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCopyFrom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mserver_req\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_extract_packet_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbytes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_server_request\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_send_message\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"<BI\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"W\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sendall_with_error_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheader\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_sendall_with_error_handle\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonotonic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m                 \u001b[0msent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m                 \u001b[0;31m# sent equal to 0 indicates a closed socket\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mBrokenPipeError\u001b[0m: [Errno 32] Broken pipe"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error in callback <function _WandbInit._pause_backend at 0x7c91d42c0e50> (for post_run_cell):\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "BrokenPipeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mBrokenPipeError\u001b[0m                           Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/backcall/backcall.py\u001b[0m in \u001b[0;36madapted\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;31m#            print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0madapted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_init.py\u001b[0m in \u001b[0;36m_pause_backend\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    417\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    418\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pausing backend\"\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 419\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterface\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpublish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    421\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_resume_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface.py\u001b[0m in \u001b[0;36mpublish_pause\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    729\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpublish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m         \u001b[0mpause\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPauseRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 731\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpause\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    732\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\u001b[0m in \u001b[0;36m_publish_pause\u001b[0;34m(self, pause)\u001b[0m\n\u001b[1;32m    347\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_pause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpause\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPauseRequest\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m         \u001b[0mrec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpause\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpause\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish_resume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresume\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mResumeRequest\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\u001b[0m in \u001b[0;36m_publish\u001b[0;34m(self, record, local)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pb.Record\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_assign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_record_publish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     def _communicate_async(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_record_publish\u001b[0;34m(self, record)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0mserver_req\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mServerRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mserver_req\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_publish\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCopyFrom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mserver_req\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_extract_packet_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbytes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36msend_server_request\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_send_message\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"<BI\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"W\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sendall_with_error_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheader\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_server_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\u001b[0m in \u001b[0;36m_sendall_with_error_handle\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonotonic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m                 \u001b[0msent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m                 \u001b[0;31m# sent equal to 0 indicates a closed socket\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mBrokenPipeError\u001b[0m: [Errno 32] Broken pipe"
          ]
        }
      ],
      "source": [
        "wandb.finish()"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "IwBW54BVeE56"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.1"
    },
    "orig_nbformat": 4,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "dba106bbaf184380bdb32b0409dad1db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c142ece118784b8d9c277bb9037be7ab",
              "IPY_MODEL_66ec6b636f1f4900b18192b11dca250e"
            ],
            "layout": "IPY_MODEL_8e50c8dad02046a59b46d5cf6b7d5779"
          }
        },
        "c142ece118784b8d9c277bb9037be7ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e3cc3be83c5944589ada73ef002c930e",
            "placeholder": "​",
            "style": "IPY_MODEL_97d49e68226d47669e5868e3d766760a",
            "value": "0.001 MB of 0.010 MB uploaded (0.000 MB deduped)\r"
          }
        },
        "66ec6b636f1f4900b18192b11dca250e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f7825c7242949ff8d545d7ca715db4e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_793dc8bf53d543e7961d4ccd0222effa",
            "value": 0.11273778440880268
          }
        },
        "8e50c8dad02046a59b46d5cf6b7d5779": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3cc3be83c5944589ada73ef002c930e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97d49e68226d47669e5868e3d766760a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f7825c7242949ff8d545d7ca715db4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "793dc8bf53d543e7961d4ccd0222effa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}