Visualizing_Function_Space_Similarity_ResNet20v1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "scratchpad",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b4022f3584e446088c45ea71b123848c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_096e5e799d674fe2b33317d06222f5b7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_50356eaf353d4460839cce8646d40d36",
              "IPY_MODEL_8ed2b4daf11b442ba66c81dee2e7cd5b"
            ]
          }
        },
        "096e5e799d674fe2b33317d06222f5b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50356eaf353d4460839cce8646d40d36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fc9034d3a9504fb6a867d10463187072",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_461939f27dbf41bdacf3084a1c1a113d"
          }
        },
        "8ed2b4daf11b442ba66c81dee2e7cd5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a08dba8b2ad44f989a65b6ab578320af",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [09:54&lt;00:00, 14.86s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e9b6dea4979c4ae583a966546b2da5c1"
          }
        },
        "fc9034d3a9504fb6a867d10463187072": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "461939f27dbf41bdacf3084a1c1a113d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a08dba8b2ad44f989a65b6ab578320af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e9b6dea4979c4ae583a966546b2da5c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e2797c5998b4c25b3abbd4371e56a1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c127897320a405c99dd89c0fab7a0b0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ac01df9d198245e18b5322e5d142631b",
              "IPY_MODEL_ef12fdb834b74a0a9414114f39d69740"
            ]
          }
        },
        "5c127897320a405c99dd89c0fab7a0b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ac01df9d198245e18b5322e5d142631b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_07b9665a266b46309304935719ca5517",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7f14de278f9441e4b03fe9020a3d84af"
          }
        },
        "ef12fdb834b74a0a9414114f39d69740": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ad1a3f26052b4eccbb28a411b3c73f1a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [00:16&lt;00:00,  2.43it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e7b3b6dbb1814cbcbf3dee455c36d781"
          }
        },
        "07b9665a266b46309304935719ca5517": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7f14de278f9441e4b03fe9020a3d84af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad1a3f26052b4eccbb28a411b3c73f1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e7b3b6dbb1814cbcbf3dee455c36d781": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10fe56316ccd4302be2fb49df1d4aaa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_710826759a7e46f398996463b5f45df0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_df69fb347c8e41f0800e6419ae5e34e7",
              "IPY_MODEL_594ae42dd45d4a749f4d626a1bc0c80f"
            ]
          }
        },
        "710826759a7e46f398996463b5f45df0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df69fb347c8e41f0800e6419ae5e34e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d80e06c932b9445dac10bf6317de56c5",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6e92c58f4fd048298f57b23c25e3656b"
          }
        },
        "594ae42dd45d4a749f4d626a1bc0c80f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f1f3f35cde634c7ca27be74b103abbff",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [01:37&lt;00:00,  2.45s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_251ade832ceb4994ac0880e1e8965acf"
          }
        },
        "d80e06c932b9445dac10bf6317de56c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6e92c58f4fd048298f57b23c25e3656b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1f3f35cde634c7ca27be74b103abbff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "251ade832ceb4994ac0880e1e8965acf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87eb3a40b6ca487dac05dead7b4ed087": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_113b0d59cd27486391963d4cf5b10d63",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_df3abf63839940cda915c08532a6a333",
              "IPY_MODEL_7efedc14d28c4ee6b45cbc9db5446e42"
            ]
          }
        },
        "113b0d59cd27486391963d4cf5b10d63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df3abf63839940cda915c08532a6a333": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bcfa853efbf942e18674a01e01fb951d",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ceca6d5c594b4f6bb3c059136723f767"
          }
        },
        "7efedc14d28c4ee6b45cbc9db5446e42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ce631a54e9d442ab9e183faf099d54b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [00:03&lt;00:00, 12.71it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ab4f8409446b4664b9aea443011ac3d7"
          }
        },
        "bcfa853efbf942e18674a01e01fb951d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ceca6d5c594b4f6bb3c059136723f767": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce631a54e9d442ab9e183faf099d54b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ab4f8409446b4664b9aea443011ac3d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "86e2122deb8b43e88af03263672e90ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5796ccdf00e7459fa6b3e01f3c4eb82c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c0f21fc88848419a9f6ab9c1e42cf823",
              "IPY_MODEL_a462d96f40334d4aa3aa79a07523696d"
            ]
          }
        },
        "5796ccdf00e7459fa6b3e01f3c4eb82c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c0f21fc88848419a9f6ab9c1e42cf823": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_010a217c2a1049ab977243fc2533825b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcfd4bdf4db0427ba483c43a67d6df62"
          }
        },
        "a462d96f40334d4aa3aa79a07523696d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fa913963a66049b389737d953763af92",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [06:37&lt;00:00, 39.71s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2751e8f7376345bd800ea45b6dbe74f6"
          }
        },
        "010a217c2a1049ab977243fc2533825b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcfd4bdf4db0427ba483c43a67d6df62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa913963a66049b389737d953763af92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2751e8f7376345bd800ea45b6dbe74f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "daaac65a8891426e804b67dddd1dc3ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fb45c39f03564158b13b5db1dd9dae46",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c44aa148bbce4b7c8496c6efabc4b73e",
              "IPY_MODEL_98bc2de090b74af0bc793b280268cde9"
            ]
          }
        },
        "fb45c39f03564158b13b5db1dd9dae46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c44aa148bbce4b7c8496c6efabc4b73e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8c7ebaf2817c4932865fb5665a4f39a6",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c484a09b31a8442c8ed395bbdcf990e4"
          }
        },
        "98bc2de090b74af0bc793b280268cde9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c6dc5b46680d4ebc8db54c5d478a6cb1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:59&lt;00:00,  5.97s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0707ebfe358d4a65acf4e9222f62769b"
          }
        },
        "8c7ebaf2817c4932865fb5665a4f39a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c484a09b31a8442c8ed395bbdcf990e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c6dc5b46680d4ebc8db54c5d478a6cb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0707ebfe358d4a65acf4e9222f62769b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "305439c4cb1547eebce197eaf29fad2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2c9b073f9b62437aafebcbae58d41821",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_14eb209382ee423ba536ac9fe8391b07",
              "IPY_MODEL_c0c97d1c43d4471ab048dd8d4a5be37e"
            ]
          }
        },
        "2c9b073f9b62437aafebcbae58d41821": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "14eb209382ee423ba536ac9fe8391b07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_88defc245af94f3ebd2ee3aaf56b8e3b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9dd9992bd3e436f9552698e7e854995"
          }
        },
        "c0c97d1c43d4471ab048dd8d4a5be37e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0dcbc1e5522b45039a54ce5fc869c8dc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:19&lt;00:00,  1.96s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3d1fd014af640828c96324f4cb88e61"
          }
        },
        "88defc245af94f3ebd2ee3aaf56b8e3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9dd9992bd3e436f9552698e7e854995": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0dcbc1e5522b45039a54ce5fc869c8dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3d1fd014af640828c96324f4cb88e61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "275cf834720e4393bd0ef4d6d3458126": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7c9faafacfac4c5ab1a1fbdce27cb9a8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8bef4c136b814da6ad3743a490900112",
              "IPY_MODEL_b39d02e7d8d14f278bcdd8d34fc883c8"
            ]
          }
        },
        "7c9faafacfac4c5ab1a1fbdce27cb9a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8bef4c136b814da6ad3743a490900112": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9c384e499e2d4a7c80f482c2dd0c882f",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_952d665b6f5a440885eda642dafc52da"
          }
        },
        "b39d02e7d8d14f278bcdd8d34fc883c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_475a3b31c90e4bfd860e6bb693f43e72",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:08&lt;00:00,  1.18it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b53854aee89646fd8fc0b387b908d879"
          }
        },
        "9c384e499e2d4a7c80f482c2dd0c882f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "952d665b6f5a440885eda642dafc52da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "475a3b31c90e4bfd860e6bb693f43e72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b53854aee89646fd8fc0b387b908d879": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayulockin/LossLandscape/blob/master/Visualizing_Function_Space_Similarity_ResNet20v1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j1LPEMCDHGgP",
        "colab_type": "text"
      },
      "source": [
        "## Set up and imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lIYdn1woOS1n",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "be2b732a-c9db-4bd2-f051-f9b8a6efd9b6"
      },
      "source": [
        "# TensorFlow Imports\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XQkwEhOpHLh7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "e54f7b7e-4dfd-492a-9d7f-d6d3c86634d8"
      },
      "source": [
        "# Which GPU?\n",
        "!nvidia-smi"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed Jul 15 16:27:31 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.51.05    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P8    29W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4LsbKhKTHNi5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install wandb"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ic6WimwMHPDS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import wandb\n",
        "from wandb.keras import WandbCallback\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IDjPCPtHHRBx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "c83ded37-dbfb-4f2b-8b63-14dfdf94c57d"
      },
      "source": [
        "# Other imports\n",
        "import os\n",
        "os.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "sns.set()\n",
        "\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "from numpy.linalg import norm"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMrBWi_qHdX6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import auth as google_auth\n",
        "google_auth.authenticate_user()"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mPBD3RhFHr_x",
        "colab_type": "text"
      },
      "source": [
        "## Retrieve the trained models\n",
        "\n",
        "**ResNet20v1 with data augmentation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8JRWnhp5HjEq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir ResNet20v1_CIFAR10_Aug\n",
        "!gsutil -m cp -r gs://losslandscape/ResNet20v1_CIFAR10_Aug/ResNet20v1_CIFAR10_Augresnet20v1_checkpoint_*.h5 ResNet20v1_CIFAR10_Aug/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EkLUQQy7INac",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e802bb20-f8f8-41e3-9bff-bdafa73596fb"
      },
      "source": [
        "model_ckpts = os.listdir('ResNet20v1_CIFAR10_Aug')\n",
        "print(len(model_ckpts))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "40\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "egQOYuPd-C7s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# https://stackoverflow.com/a/2669120/7636462\n",
        "import re \n",
        "\n",
        "def sorted_nicely(l): \n",
        "    \"\"\" Sort the given iterable in the way that humans expect.\"\"\" \n",
        "    convert = lambda text: int(text) if text.isdigit() else text \n",
        "    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] \n",
        "    return sorted(l, key = alphanum_key)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVirk69w-rSz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "e3739864-c780-4f70-9372-2a087b2c030f"
      },
      "source": [
        "model_ckpts = sorted_nicely(model_ckpts)\n",
        "model_ckpts[:5]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ResNet20v1_CIFAR10_Augresnet20v1_checkpoint_0.h5',\n",
              " 'ResNet20v1_CIFAR10_Augresnet20v1_checkpoint_1.h5',\n",
              " 'ResNet20v1_CIFAR10_Augresnet20v1_checkpoint_2.h5',\n",
              " 'ResNet20v1_CIFAR10_Augresnet20v1_checkpoint_3.h5',\n",
              " 'ResNet20v1_CIFAR10_Augresnet20v1_checkpoint_4.h5']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4WsFvfwYIeqW",
        "colab_type": "text"
      },
      "source": [
        "## Create data loaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3h2SZFvaIZzp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "beb5cd25-7061-4147-a876-fa28222842c4"
      },
      "source": [
        "# Load the training set of CIFAR10\n",
        "(_, _), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
        "\n",
        "y_test = y_test.flatten()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 6s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F0FvM8jUIn3r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 128\n",
        "\n",
        "def normalize(image, label):\n",
        "    return tf.image.convert_image_dtype(image, tf.float32), label\n",
        "\n",
        "\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
        "test_ds = (\n",
        "    test_ds\n",
        "    .map(normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        ")"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3eLjiAtjI5Wl",
        "colab_type": "text"
      },
      "source": [
        "## Disagreement of predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70h3set2I2FF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(test_dataloader, model):\n",
        "  true_labels = []\n",
        "  pred_labels = []\n",
        "  \n",
        "  for imgs, labels in iter(test_ds):\n",
        "    preds = model.predict(imgs)\n",
        "    \n",
        "    true_labels.extend(labels)\n",
        "    pred_labels.extend(np.argmax(preds, axis=1))\n",
        "\n",
        "  return np.array(true_labels), np.array(pred_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CD9-FruvJK_X",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "b4022f3584e446088c45ea71b123848c",
            "096e5e799d674fe2b33317d06222f5b7",
            "50356eaf353d4460839cce8646d40d36",
            "8ed2b4daf11b442ba66c81dee2e7cd5b",
            "fc9034d3a9504fb6a867d10463187072",
            "461939f27dbf41bdacf3084a1c1a113d",
            "a08dba8b2ad44f989a65b6ab578320af",
            "e9b6dea4979c4ae583a966546b2da5c1"
          ]
        },
        "outputId": "2bd3d3dd-3f5f-4343-a860-2700da2063ce"
      },
      "source": [
        "predictions = []\n",
        "for i in tqdm_notebook(range(len(model_ckpts))):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model('/content/ResNet20v1_CIFAR10_Aug/' + model_ckpts[i])\n",
        "  # get predictions for model\n",
        "  _, preds = evaluate(test_ds, model)\n",
        "\n",
        "  predictions.append(preds)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b4022f3584e446088c45ea71b123848c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9C9i-HyKlea",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "3e2797c5998b4c25b3abbd4371e56a1f",
            "5c127897320a405c99dd89c0fab7a0b0",
            "ac01df9d198245e18b5322e5d142631b",
            "ef12fdb834b74a0a9414114f39d69740",
            "07b9665a266b46309304935719ca5517",
            "7f14de278f9441e4b03fe9020a3d84af",
            "ad1a3f26052b4eccbb28a411b3c73f1a",
            "e7b3b6dbb1814cbcbf3dee455c36d781"
          ]
        },
        "outputId": "e9c8ea3d-1127-4fe1-a840-01cefa3ab15c"
      },
      "source": [
        "empty_arr = np.zeros(shape=(40, 40))\n",
        "\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  preds1 = predictions[i]\n",
        "  for j in range(i, 40):\n",
        "    preds2 = predictions[j]\n",
        "    \n",
        "    # compute dissimilarity\n",
        "    dissimilarity_score = 1-np.sum(np.equal(preds1, preds2))/10000 \n",
        "    \n",
        "    empty_arr[i][j] = dissimilarity_score\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = dissimilarity_score\n",
        "\n",
        "dissimilarity_coeff = empty_arr[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e2797c5998b4c25b3abbd4371e56a1f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U_Li-cbmMiCX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 485
        },
        "outputId": "726ed15e-12d4-45cc-f513-6453e8599841"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns_plot = sns.heatmap(dissimilarity_coeff, cmap='RdBu_r');\n",
        "plt.xticks([5,10,15,20,25,30,35],[5,10,15,20,25,30,35]);\n",
        "plt.yticks([5,10,15,20,25,30,35],[35,30,25,20,15,10,5]);\n",
        "plt.show()\n",
        "\n",
        "figure = sns_plot.get_figure()    \n",
        "figure.savefig('prediction_disagreement.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAHUCAYAAAAwb/F2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3Rc9X3v/c/MaEZXj2XJliyMgwOJQQGS0ObSlkIabuY5FdceAseF0pNipyE2fULBdlOwsHFaRDnQBOynq6RQc0ga4tCTgEJLSjhZDe4JJcFJQxxTMBdjI0tgWZJ1HWlmP39wcLYie/QRjKS5vF9rzVry8GXv396zZ37z3bPnM5EgCAIBAICiF53tAQAAgJnBpA8AQIlg0gcAoEQw6QMAUCKY9AEAKBFlM7myv6s7adKaj5zzXmtZ7/+9M626xJn/1aqLjA1bdYP177PqyqIRq06S3hgcs+oaKmNW3T/uOmDV7e8fsepqK+NW3bwKr+4n+3qtugP9Kavu/QtrrLqf7+uz6lwNc8qtuqd2dVl1oyNpqy49lrHqyiu9p/fwwKhVFzGP6ZEhb3mJcm98ZQnvuK+rq7Tq9rx00KqTpIE+73XBFYn4rwuOcvO5OTbqHVuuION96Wt4YNCq69jxhFWX2nGfVfdu/XFkSc6X+TfBKzlf5jtBpw8AQIlg0gcAoETM6Ol9AADyXSy3n8LkFSZ9AABCYjm+9iKfcHofAIASQacPAEBIMZ/ep9MHAKBE0OkDABBSzJ/pM+kDABDC6X0AAFDwZrTTd06ZRN23WFEvmtMVRL1dkUp78ZNTeaOYNiMtR70yHUp5kZtDZp2rKu49Ju56U2kvbtZdXu+gF+tbmfCOhZ5BL252LOVtR8Y9Dka82OZ4ufd4pM39HAm8ozoIvO1w15sZ8ZY3OOztFzfGWJJiMa8vsmNuzTbLjbl1j4Vcx/BGzf3ixus2nXbOuxlOzhXz6X06fQAASgSf6QMAEMJn+gAAoODR6QMAEJIPn+m//PLLWrdunXp6elRbW6u2tjYtWbJkXM3dd9+tr33ta2poaJAk/dqv/ZpaW1uzLpdJHwCAkHw4Bd7a2qrly5froosu0re//W2tX79eDzzwwIS6iy++WGvXrrWXmw/bBgBAUevr69PevXsn3Pr6+ibUHjhwQDt37lRLS4skqaWlRTt37lR3d/e7HgedPgAAIdNxen/r1q265557Jty/atUqrV69etx9HR0damxsVCz21tduY7GYGhoa1NHRobq6unG13/nOd/TUU09pwYIFWr16tU477bSs42DSBwBgml199dW65JJLJtyfTCbf8TKvuOIK/fEf/7Hi8bi2b9+ua6+9Vo899pjmzZt31P+HSR8AgJDp+MpeMpm0J/impiZ1dnYqnU4rFospnU6rq6tLTU1N4+oWLFhw+O/TTz9dTU1NeuGFF/Sxj33sqMvmM30AAEJikUjOb1NRX1+v5uZmtbe3S5La29vV3Nw84dR+Z2fn4b9/8YtfaN++fXrve9+bddkzHMM7eU3ajFRNDw1adUHEe18TMevcd4CxaO7fKibSI1ZdRVlu38vVVyWsul4zBnVBstyq6zrkba8bY5ww98tQytuOykRuY24TZmyuG9EamGmzEfNYddcbMV/g3OVVmMdfbY15nFbHrTrJf+xy/UIaiXv7sMyMvo6ax35qyIuWfvXfHrXqjvutC6y66DS8Xha6W265RevWrdOWLVuUTCbV1tYmSVqxYoWuu+46nXrqqbrzzjv185//XNFoVPF4XLfffvu47v9IOL0PAEBIPiTynXDCCdq2bduE+++9997Df7/9RmAqOL0PAECJoNMHACAkHxL5pguTPgAAIflwen+6cHofAIASQacPAEBIMZ/ep9MHAKBE0OkDABBSzJ/pM+kDABBSzJM+p/cBACgRM9rpp8zYTUc07g09kvEiVV3uJgS529TDRmNefO3wWK9V58bX9gx70Zxzyr3HpHfQW16upcbMXNocL8+NGM2Yj4cbDRs3Y33dKNexUS8i2+XG/w4Ppqy61FiFVTcy5L8mRM0LuoZHvGW6x4K7b9LmMRiYL0ivPf0dq27xx3/XqsuYx2qsLL9OOnMhHwAAKHj59fYKAIBZxmf6AACg4NHpAwAQUsyf6TPpAwAQwul9AABQ8Oj0AQAIKebT+3T6AACUCDp9AABCivkz/byb9IO0lxwViXopYraId9LDTRVMpdKqjXurjpnpW2M5TDSUpB4zGc8dX22Ft8G5Xq+b8Ddippe5SYXu+EaGvPG56WpuqtuwuV8yOU4qdMeX64S/vkMjVl1dY7U693iplW4yXq5TFzMpb1tcHTuesOre85stVl3cTN9Mmcd+vuH0PqbMnfABzCx3wgeKUd51+gAAzCb3NxcKEZ0+AAAlgk4fAICQSBFfycekDwBASLSIJ31O7wMAUCLo9AEACInEircfLt4tAwAA49DpAwAQUswX8tHpAwBQIrJ2+tu3b9fpp58uSTp06JA2btyoHTt2qLm5Wa2trZo/f/6MDPJIIuUVuV1gZswqC9wkXDPWV5Ki8uJI3RRe901qeVlu3/O5gRZufG3CHJ+7PLfO3S/2enP8+aAbX+tGpQbmgeVG0rqxvuVmbPPIsBflmhrx9osbdyxJFdUJqy7XUcbDYymrzo3XbTrtHKsu4j6HzWO6LO7Fpbt1M6Vkr96/4447Dv991113qbq6Wlu2bNHxxx+vTZs2TfvgAACYaZFoNOe3fJG1FQhCbe2Pf/xjffOb31Q8HtfSpUt1wQUXTPvgAABA7mSd9FOplHbv3q0gCBSJRBSP//J0XDSP3rkAAJArxXx6P+ukPzw8rBUrVhz+d2dnpxobG9Xf38+kDwBAgck66T/55JMT7uvt7VV5ebm+/OUvT9ugAACYLSX7lb1du3bp0ksv1WWXXabdu3dr5cqVOvPMM7Vs2TINDAzM1BgBAJgxkVg057d8kXUkmzZt0uc+9zn9/u//vq655hq1tLTopz/9qVpbW3XbbbfN1BgBAEAOZJ30BwYGdPbZZ+viiy+WJF144YWSpLPOOks9PT3TPzoAAGZYNBbJ+S1fZJ30w1/Zezuk522ZTG7DKAAAwPTKeiHfokWL1N/fr5qamnFhPPv371dlZeW0Dw4AgJnmpk8WoqyT/ubNm494fzKZ1JYtW6a8srQbYWsIRoa9QjcO16xzz9IEZpyl5O+XhLnyXO5nyY/DHTXP/rjxtakRb3kjZgRqmbled3mVidn5vaogxyfZ3Fhfd70ZM9bXrXMjWssrvccjNoX46dSQGwHsxXinU0NW3f6f/m+rzo3XdQVmzrh7zLh17rEwU6J5dOFdrr2jV62qqipVVVXleiwAAGAa8dO6AACElOz39AEAQPGg0wcAIIROHwAAFDw6fQAAQrh6HwCAEsHpfQAAUPDo9AEACIkWcSIfnT4AACViRjt9Jx42nfJiG9MjI95KM148pqLernCjcCMZbzskadSMoIya0b4ZM0qzMuHFm7p1sSlED+dSbVXcqnvpjdn5kaioGftaZu7nwDxe3Fhad3lu1GxZ3NvesVEzttl8lSovN2N4p3CR1pj5PM6Mpaw6N1534Yc+adVVVHvJqO5j5x4Lbl2hZthHuJAPAIDSkE8/hZtrxft2BgAAjEOnDwBACF/ZAwAABY9OHwCAEC7kAwCgRHAhHwAAKHh0+gAAhBRqvoCDTh8AgBJBpw8AQAg/rZsjlcbFEfGahLWsWHm5VRfEK626yOiQVZdysoTfWqJZJ7nXjKTSXmypWzdkRh73D3sRnlVxL0Y2NeaNr2fQizZtmOMdCzHzlJ27vbVeAqptzHw8MmYEqhtzOzI8atW5hge8/eeeQq0wY5bdx3ds1I/Idms7djxh1TWddo5VFy3zXgdd7g/IuMeWy43r5ZzzzKHTBwAgpJjDeZj0AQAIKebv6RfvlgEAgHHo9AEACIlEi7cfLt4tAwAA49DpAwAQwlf2AAAoEVzIBwAACh6dPgAAIXT6AACg4M1op++EHMXi3vuQ9MiIVRcPzBjIwIssLS/zxjeVQKdoxIwjNRdaV+lFePYOevGrbszt3HLvcKo3o5bdOOE6c3m1h7y6qoQXJ3xcvZfDu3tfr1XnhddK5VEvljZiHlflFd7ycs18yilqPufSZuRrotLf3pefesSqW/zx37Xqysyoape7Le4+dF+LMubrqjvBpM3n+kzJh6/svfzyy1q3bp16enpUW1urtrY2LVmy5Ii1L730ki655BItX75ca9euzbrc2d8yAADySCQWy/ltqlpbW7V8+XI9/vjjWr58udavX3/EunQ6rdbWVp1zjve7DnymDwDANOvr61NfX9+E+5PJpJLJ5Lj7Dhw4oJ07d+r++++XJLW0tOjWW29Vd3e36urqxtX+7d/+rX7nd35Hg4ODGhwcnHQcTPoAAIRMx4V8W7du1T333DPh/lWrVmn16tXj7uvo6FBjY6Ni//cMQSwWU0NDgzo6OsZN+rt27dJTTz2lBx54QFu2bLHGwaQPAMA0u/rqq3XJJZdMuP9Xu3zX6Oiobr75Zv3lX/7l4TcHDiZ9AABCotNwId+RTuMfTVNTkzo7O5VOpxWLxZROp9XV1aWmpqbDNW+88Yb27NmjlStXSnrr44MgCNTf369bb731qMtm0gcAII/U19erublZ7e3tuuiii9Te3q7m5uZxp/aPOeYYPf3004f/fffdd2twcJCr9wEAmIpILJrz21TdcsstevDBB7Vs2TI9+OCD2rBhgyRpxYoV+tnPfvaOt41OHwCAkHxI5DvhhBO0bdu2Cfffe++9R6z/1YsBj2b2twwAAMwIq9M/ePCg9u/fL0lauHCh5s2bN62DAgBgtuRDIt90yTrp79mzRzfffLN27typhoYGSVJXV5c+8IEPaMOGDUeNBDyalBGTmR714hhj1TXeSs2sz0jaC0HtT5nRsJX+VygSZrzum0Npq64/5W2LG1v60hsDVt0HFs6x6l49MHmAhCTVVXuxuR09w1Zd/7AXOzxm75d+qy4a9R7f0RHvcYuYy3O5kaox85SnG/8bi+d2v/SZx8Hz//KwVSdJJ577e1Zdasg7ttwYXjeWdnggZdXF3Bjecm98lZXeJ8PDA95+qTDHh3cv6yO3Zs0aLV++XPfff//hrzBkMhk9+uijWrt2rR566KEZGSQAADMlHz7Tny5Zt6ynp0cXXnjhuO8sRqNRXXTRRert9X5EBACAQpIPV+9Pl6wjqa2tVXt7u4LQ6b8gCPTII4+84xQhAAAwO7Ke3r/tttvU2tqqjRs3qrGxUUEQqLOzU83NzbrttttmaowAAMyYaB515rmWddJfsmSJtm7dqu7ubnV0dEh6Kx7wV3/lBwAA5L+sb2d27dqlSy+9VCtXrlRFRYW+9KUv6ZOf/KQ+8YlP6Be/+MVMjREAgBkTiUZzfssXWUeyadMmfe5zn9OVV16pa665Ri0tLfrpT3+q1tZWtbW1zdQYAQCYMSV7Id/AwIDOPvtsXXzxxZKkCy+8UJJ01llnqaenZ/pHBwAAcibrZ/rhq/ZPP/30cf8tk/HCIwAAKCT51JnnWtYtW7Rokfr730od27Rp0+H79+/fr8rKyukdGQAAyKmsnf7mzZuPeH8ymdSWLVumvLL+scnjPseGvMjNIOVFbkZHvKjUSNqLi3Qjc8sOdVl1klQ7p8GqM9NSdUqDF4ebNhdYYb7rXTzXeyP4wcVzrbpkuRf1+aYZRVpf48X6psbMKGgzDrerb8Sqc78mNNfcjp5D3npr55RbdS433nlOhff4dprxuj+4/36r7tcv+32rTpKStRVWnXssuPtmxIweHjFfLxNmvG7SPBYSZmxuT7/33Kwyj4WZkk8X3uXaO9rTVVVVqqqqyvVYAADANMqvt1cAAMyyaMz/wbRCw6QPAEBIyV7IBwAAigedPgAAIXT6AACg4NHpAwAQwlf2AAAoEZzeBwAABW9GO/2asslTqxI1cWtZkTIvlSww6zIVSavOlamaZ9fu7/fSACvMFKxnO3qtum4zLctNEUubiYGvvjmY0/W6yW7/2XnIXJ53DKbN358YOOTt54j5Fryza8Cqc1PYes3jYGw0bdVFzXS67u4hq+6Zhx606s747//dqpOk3gPeMdhnpgFmzBTHdDq3v1niPiZDXjCpBsz0yIoq73V1ZNh7bRs20zdnCp0+ABQJd8IHilF+vb0CAGCWcSEfAAAlIhIt3hje4n07AwAAxqHTBwAgjE4fAAAUOjp9AADCivhCvuLdMgAAMA6dPgAAIZFY8X6mz6QPAEBYEV/IN6OTfqURbRiv9uIdx/q9XMl4vNKqi6S9uEhXKuLv2mjEi9KsNGKMJSlmxqAOpbz1LkiWW3WZwIzrNeN1O3q9mNaqRI1Z5z2Ru/q86NXjF3jrdWXMHOOyuPep3NioF/maMmN4Y2Y06ciQ91z6+WPbrLqPXn6lVddYW2HV9bzhxRhL0qAZS5s2Y3hjZpR2NOY9h8vNyGg3Dtc11O/tl4j5WjRmvhbh3aPTBwAgrIg7fS7kAwCgRNDpAwAQQvY+AAClgtP7AACg0NHpAwAQRqcPAAAKHZ0+AAAhXMgHAECp4PQ+AAAodHnX6QdpL86yrMaLQA3KvFjfyJgXK+kyk2anpMKM5qww41ITZiRo0oz6rDCX56633KyrqfAO49QB79iaYy7P3Q6XG1/rxuFGzfGlhsasOjf+143XPfm/XJbT9R4a9rYjM4UnZ8R8iANzme62pEa8WNpo1Ktzjxk3Ttjlvp7ner3vGp0+AAAodHnX6QMAMJuK+ad16fQBACgRdPoAAITxlT0AAEoEF/IBAIBCR6cPAEBIhE4fAAAUOjp9AADCuJAPAIDSUMyn97NO+i+88ILe//73S5JGR0e1ZcsWPfvss2pubtaf/MmfqLKyckorSweTR1WmU16sZK4FcW9bxsy4TTMxV5I0bEZVSt6BeNCMcx009/XwmFfXN+ytt7bKi/Xd3ztk1XX15TZC+c0cL29s1Nt/5ZXefnHjeuPl3nv6qHmw/qz9G1bdqS2fsurc8bmxubGotx3ufpb8eF2Xuy1Rc1vcqOWMGXNblvBeY9z9EjG3w40JxruXdU+vWbPm8N933323nn/+ef3hH/6henp69Bd/8RfTPjgAAGZcNJb7W57I+lY7CHXmP/jBD/Tggw+qurpaZ5xxhi6++OJpHxwAAMidSSf94eFhBUGgWCym6urqt/6nsjKVlXE5AACgCJXqhXzPP/+8TjvtNAVBoEgkos7OTjU2NmpkZESZTJ79FCIAADlQzD+4k3XS37Vr1xHvHxkZ0caNG6dlQAAAYHpkPYdx8OBB3XTTTfr0pz+tr371q4fvTyaT+ru/+7tpHxwAADOuiC/kyzrpt7a2KplM6oorrtATTzyhVatWaWxsTJK0d+/eGRkgAADIjayT/iuvvKI1a9bovPPO03333acFCxboM5/5jEZGRsZd2Q8AQNEo1U5/dPSXISCRSEStra1aunSpVq5cqZGR3AaYAACQDyLRaM5v+SLrSBYvXqxnnnlm3H1r167Vhz70Ib3yyivTOS4AAJBjWa/ev/322xWJTIxRvP7663XhhRdOeWU1RmRkzIyBjJRXeCuNeO+wIqkBqy4VrbPqElPI4a2Je9s8Zn6iUmnuw5QZzenWJSu8eNOOnmGrLm1GfTrxzpJUmfCyJcoTY1ZdwoxAdfWb+6XCjDEe6POW9/PHtll1J/+Xy6y6sVHveImWeXWV8YRV50rbsddS9Aivf+9GwoweHjEjrXMdE1zmvhaZ0dLu+IJInn1cnEen43Mt6xFYW1t71P/2vve9L+eDAQAA04dYPQAAwswzxIWoeLcMAACMQ6cPAEBYEXf6TPoAAIQERTzpF++WAQCAcej0AQAIK+JOn0kfAIA88/LLL2vdunXq6elRbW2t2tratGTJknE1Dz/8sP7+7/9e0WhUmUxGl112mf7gD/4g63KZ9AEACMtxKNM70draquXLl+uiiy7St7/9ba1fv14PPPDAuJply5bp0ksvVSQSUX9/vy644AJ97GMf00knnXTU5TLpAwAQNg1Z+X19ferr65twfzKZVDKZHHffgQMHtHPnTt1///2SpJaWFt16663q7u5WXd0vU2FramoO/z08PKzR0dEjpuiGzeikHzd2ZDTuDSkYTXkrTXuRqgr8aE5H5RQiWiPy1p02kyrnmHGzQ2aU5lDKq5tX6cXDlkW9d9F1NeVWXe+gdywcV19t1aXGvO3NtXi5F/2ZqPQe3x3/6x+sOjdetzrpRV+7MbdR8zjImMvrOeT9CFgs5j833VjaMjP6uixuxoKb+8blPibu+Nx96H407h4LhWzr1q265557Jty/atUqrV69etx9HR0damxsVCz21nEVi8XU0NCgjo6OcZO+JH3ve9/TnXfeqT179uhP//RPdeKJJ2YdB50+AAAh0/GVvauvvlqXXHLJhPt/tcufqrPPPltnn322Xn/9dX3uc5/TmWeeqeOPP/6o9Uz6AABMsyOdxj+apqYmdXZ2Kp1OKxaLKZ1Oq6urS01NTUf9f4455hideuqp+v73v5910i/e7yUAAPBORKK5v01BfX29mpub1d7eLklqb29Xc3PzhFP7u3fvPvx3d3e3nn76aS1dujTrsun0AQAIy4Pv6d9yyy1at26dtmzZomQyqba2NknSihUrdN111+nUU0/VQw89pO3bt6usrExBEOjKK6/Ub//2b2ddLpM+AAB55oQTTtC2bdsm3H/vvfce/vsLX/jClJfLpA8AQFgedPrTpXi3DAAAjEOnDwBACL+yBwAACh6dPgAAYUXc6c/opP/a4OikNU0dh6xlReIJry49+TolKUh4Ea0xMy0yHZiZuZL2D3hRwcfUeA/XATOWtr7a24epsdzGBLvc9S6cW2nVPfNyt1VXU+Ht53TG22A3AjUwl/f01/6nVffx5VdZdakh7/gbNo8rN6K1zIwTTo14sciNDd5zeF+fF9c7FSND3uuMGzc7NuodM+7c5D4mw8Zr9FSWNzriHVvx8jzrP/PgB3emS/G+nQEAAOPk2dsrAABmWRGf3i/eLQMAAOPQ6QMAEFLMX9lj0gcAICxavJN+8W4ZAAAYh04fAICwIj69X7xbBgAAxqHTBwAgrIg7/Rmd9OsSk+/IF5/epw+cvWTyun/4Z73vv50/ad3Y04+q7DcvdoanSGpg0pqFo69pX8XiSeu6h9JqqPZ2b9LYL5KfeFdf5SXtvfrmoFVXV+Mtryoes+p6zPSySnN5bnJfQ7LcqhtMeQlwtVVxq+51c3w/a/+GVXdqy6esusG+EUXMBLhRM/WuprbCqnMS6ob6UyqvnHwfRqMRRY0ozAMHh1RmHDM1cyt06ODQpHWSFIl7+y9hJsqVJbxj2k34c6XNY7DKPKbdZEH3+HOXN2OKeNLPuy1zJnxJ1oQvKacTviRrwpdkT/jAdMnnCV+SNeFLsiZ8SdaEL8me8IFixMwEAEBIMX9Pv3i3DAAAjEOnDwBAGJ0+AAAodHT6AACERfLs2wQ5xKQPAEAYp/cBAECho9MHACCEr+wBAICCZ3f6fX19kqRkMvmOVxabhYsjgpi3iUHlXHOBXtlYxiyUNGzm69ZVeu/RDqXGrLrugRGrrt6M4d3X6yWd1ZiRpS+90W/V1dfkNl7XjfU90J+y6nIdrxst846DWMyrc5Ps3CjXCjMG2hV4q1XCjLitb6zRsJkamDGfx94zzl+e+9jlWsZ8LRob9R4U+xg0o8hnTBF3+llffbu7u3XHHXfon/7pnyRJQRAoGo3q/PPP1w033KC6uroZGSQA5Io74aN0BUV89X7WtzM33nijFi9erCeffFI7duzQT37yEz3xxBM69thjdeONN87UGAEAQA5knfT37dunz372s5o3b97h++rq6nTttddq79690z44AABmWhDk/pYvsk765eXl2rFjx4T7n332WSUSuf3cDgAATK+sn+lv2LBBa9asUXl5uRYtWiTpre5/ZGREbW1tMzJAAABmUiafWvMcyzrpf/jDH9bjjz+u5557Th0dHZKkpqYmnXLKKYoU8YUOAIDSVbxT/iSn9w8ePKibb75Zd911l7q6unTeeefp1FNPVSQS0erVq2dqjAAAIAeyTvqtra1KJpO64oor9L3vfU+rVq3S2Nhb30h97bXXZmSAAADMpEyQ+1u+yDrpv/LKK1qzZo3OO+883XfffVqwYIE+85nPaGTEC3UBAAD5I+ukPzr6yxCLSCSi1tZWLV26VCtXrmTiBwAUpSAIcn7LF1kv5Fu8eLGeeeYZffSjHz1839q1a3XnnXfq3nvvnfLKhtKTRzceet2LXlXGi1SNDh70lhfzvoI4FK2x6jKBFwkqSdVxL/Kx34yR7R/2QkGb5lbmdHmuhqQXm5trCTO+tnfQi9f95l3/n1V3zmeusercuOOeQS9RrrYqbtW53HjiRfOqcrq83iHv8Xh/4xyr7oXOQ1adJB2/wHu+v3pg0F5mLqUz3j5Mm+eXKxNeRLb7XHKj13N9rOLosj7Ct99++xGv0r/++ut14YUXTtugAACYLfn0GXyuZZ30a2trj/rf3ve+9+V8MAAAzLYinvP5aV0AAEqF/dO6AACUgmI+vU+nDwBAiaDTBwAgJJ++YpdrTPoAAIR4X4QsTJzeBwCgRNDpAwAQUsRn9+n0AQAoFTPa6aeNd0+Jai+Ocayv16ori3uRoDLjIhNRr84skyQNm3Gk8yr8aF/HiLneY+u8uF43VrWjZ9iq6x3y4mZjU9nZBjde979+/rNWnRube8iMO3a3141edcc3YsZAu9ztbUhWWHXu9g5OYTteesOLBe/q836LpDKR2+dwuRmH626zuw9dbqxvyohon0nF/JU9Tu8DABBSzFfvc3ofAIASQacPAEBIfn3YkFt0+gAAlAg6fQAAQor4I306fQAASgWdPgAAIZkibvWZ9AEACCneKZ/T+wAAlAw6fQAAQkjky5E3RiaPghw1ozkjMe8kRXS4z6pLz22y6qoi3noTgbcdkvTmoFeb6xje4+Z7EcVDZoRnXXXCqst11Of7F9ZYdddfe6tVd9Xa1VZdIscRqFVmRKsb+drclLvKObAAABuLSURBVLTqEmVeLPJQyj+mHfU13vFyaNiLCXYfj7IpxDYnyrzHpKbCeyl1j303XnfMXF6uo6rdfe0eM4ky71jAu0enDwBASBFfx8ekDwBAWKaIL+XjQj4AAEoEnT4AACHFfHqfTh8AgBJBpw8AQAhf2QMAoERweh8AABQ8On0AAEL4yh4AACh4M9rpv68mPmlN5bwKa1nRhFeXqZxr1UVGBqy6NzLlVl1trR8rubTeexhGxjJW3XwzDvf/vNRt1Z3UNMeqe+OQFw/rcmNa3XjdO7fcbNXteLXHqquMexGt9TXeMeMub+HcSqvu1QPeMT23ytvPNeXecdqQ9La334zcdiNk3bpF87z4aUlqqvVeZ7r7U1ZdpRm13DPoRQ+7cb1u/O/cqslfo6fCfUzc/TJT+EwfAAAUPD7TBwAgJFPErT6TPgAAIWnvk9SCxKQPAECeefnll7Vu3Tr19PSotrZWbW1tWrJkybiazZs367HHHlM0GlU8HtfnP/95nXHGGVmXy6QPAEBIPpzeb21t1fLly3XRRRfp29/+ttavX68HHnhgXM0HP/hBffrTn1ZlZaV27dqlK6+8Uk899ZQqKo5+ASoX8gEAMM36+vq0d+/eCbe+vr4JtQcOHNDOnTvV0tIiSWppadHOnTvV3T3+G1dnnHGGKivf+jbPiSeeqCAI1NOT/dtHdPoAAISkp6HT37p1q+65554J969atUqrV68ed19HR4caGxsVi731VcZYLKaGhgZ1dHSorq7uiMv/1re+pfe85z1auHBh1nEw6QMAEDIdp/evvvpqXXLJJRPuTyaT73rZ//7v/64vfelLuu+++yatZdIHAGCaJZNJe4JvampSZ2en0um0YrGY0um0urq61NTUNKF2x44duvHGG7VlyxYdf/zxky6bz/QBAAhJZ3J/m4r6+no1Nzervb1dktTe3q7m5uYJp/b/4z/+Q5///Of15S9/WSeffLK17Bnt9OPRyd9j1DSZsbnlXjxmUF7jLS81aNXNM6NSu4bSVp0kjYx5p5LeU+UdOa/3Dlt1bjSnG7/qLq/XjBj9HzfdYdXddPtaq+6F/f1WnbsdrgP9XjzxsWY8bO+Qt//ceN1uc3xupGr3gLc8N1a6yYwdPmFBtVXX1efHRb/Q6R0z7mNcaz4mroQZw5sy93VXjqO03fElYvSfv+qWW27RunXrtGXLFiWTSbW1tUmSVqxYoeuuu06nnnqqNmzYoOHhYa1fv/7w/3f77bfrxBNPPOpys76ab9++Xaeffrok6dChQ9q4caN27Nih5uZmtba2av78+bnYNgAA8kY+fGXvhBNO0LZt2ybcf++99x7+++GHH57ycrO+vbrjjl92WnfddZeqq6sPf26wadOmKa8MAIB8lw6CnN/yRdZOPwgN9Mc//rG++c1vKh6Pa+nSpbrgggumfXAAACB3sk76qVRKu3fvVhAEikQiisd/+bOLUePzeQAACk2OL+vJK1kn/eHhYa1cufJwx9/Z2anGxkb19/cz6QMAUGCyTvpPPvnkEe+PxWK6++67p2VAAADMplx/gyefvKN2vbKyUp/97GdzPRYAADCNsnb6L7744lH/28GDB3M+GAAAZls+fGVvumSd9FtaWrRo0aJxV/G/bbJf8gEAoBCli3fOzz7pL1q0SF/72tfU2Ng44b994hOfmLZBAQCA3Ms66Z933nnat2/fESf9c889d8or6xudPJp26IAXexmMeFGz0f43rLpIZsyqS1cd+WcNf1VjlRfXK0mDZgzv831elGZdjRf12WPG4f7H3l6r7ryTGqy6y6/6c6vuTzfdYNW56s394mpIlud0ef0j3jFYX53b7ag0o6Vrq+KTF8k/rtxoWDf+t8/cf90DKatOko6r96KR3RhZN5bW5V5wFot4+9B9jOeadW+YkceVCf/1ciaU7On9tWuPnml+00035XwwAABg+vDTugAAhBTzV/aY9AEACCnm0/vE6gEAUCLo9AEACCnmr+zR6QMAUCLo9AEACCnmz/SZ9AEACMkU8dX7nN4HAKBE5F2nn+o307wOetn/8ZiXXpapnGvVNalPHUFy0rrOwbTmlnvvqXpGJk8qlKSTqr19861+L3HsuPle2lhjjZc85ybtPfQ/v2jVPf+Gl87oJrstmONth5sA123uZze577S5tVbdoJFsKUkxbzPsi5b6hr3j77g677hytyNqpslJ0gcW1Exa89FFtXqtd8haXrmZoNdrpgG6yXhp8/RyPOqNzz1d7a531DxoFtdWWnXlZqLhTOFCPhzmTPiS7AkfQG44E74ke8IHilHedfoAAMymYr6Qj3YUAIASQacPAECIe21DIWLSBwAghK/sAQCAgkenDwBACF/ZAwAABY9OHwCAkGL+yh6TPgAAIVy9nyNOLGiiJm4tq3LpyeZKveVFUl5KV2+m2qqrq4xZdZJUFfc+ZfnOa14M6lDKizftGfSWd+OqTVbdA39/q1X33V1dVt0Hj/WikfvNCNTuAS82N2FGr7rcmOB9ZlJc/7C3ve52VCa8Y9Vd3rC5ve7j1msep27ssCS9cmDQqjt+gfd87zo0YtXVVHgvue5z2D22XO74aiu819XX+4a95VV6y8O7R6cPoKS4Ez5KV5qv7AEAgEJHpw8AQEgxd/pM+gAAhBTzpM/pfQAASgSdPgAAIXT6AACg4NHpAwAQUsydPpM+AAAhxTzpc3ofAIASMaOdfm1txaQ1lfU13sIyXkxlpiJp1UXSXkRrZsB7B9g34sdjmim8+o1jvW35WUefVfc/brrDqlt1y/VWXWe/F0VaGfdiX59+qduqi0W9/NWTF3n779U3vcS2cjOW9oXOQ1ZdXXV5Ttfb0etFoLqPhxvDW2XG+roRsm5M8HDaW557vEjS3oNeNHKuY3Pd5blGzPW6Ha4bBV2o6PQBAEDB4zN9AABC6PQBAEDBo9MHACCkmDt9Jn0AAEKKedLn9D4AACWCTh8AgJAxOn0AAFDo6PQBAAgp5s/0mfQBAAgp+Un/4MGD2r9/vyRp4cKFmjdv3jta2WDf5DGtqT4v9jIz6EWbxsa8KFJXpZmZe2gKMZrDZkTm/ErvPdqmNW1W3U23r7Xq3EjQ3sFRq+74hmqrrrY/btUdM3fyeGdJenPAi1puMuKip+I986usujkJ7/E9MOhtx8nHeLHDh1JepKp7HMyr9B43NzbXfQH+UKO3vc9FvNcOSaoxH5PuIe8xScS8149oxIsKjpmJwukcz2EVZiSz+9o2x4xaxruX9Yjes2ePbr75Zu3cuVMNDQ2SpK6uLn3gAx/Qhg0btGTJkpkYIwAAMyYdlGinv2bNGi1fvlz333+/otG33tllMhk9+uijWrt2rR566KEZGSQAAHj3sp6j6enp0YUXXnh4wpekaDSqiy66SL29vdM+OAAAZlo6E+T8li+yTvq1tbVqb29XEDrVEQSBHnnkESWT3udnAAAUkmKe9LOe3r/tttvU2tqqjRs3qrGxUZLU2dmpk046SbfddtuMDBAAAORG1kl/yZIl2rp1q7q7u9XR0SFJampqUl1d3YwMDgCAmZZPnXmuWd9HqaurmzDRX3DBBXr00UenZVAAACD3sk76L7744hHvD4JABw8enJYBAQAwm9IZL1+gEGWd9FtaWrRo0aJxF/K9raenZ9oGBQAAci/rpL9o0SJ97WtfO3wRX9gnPvGJaRsUAACzpWQ/0z/vvPO0b9++I07655577pRX1js6+SmTN58/YC1rUWeHVVf+zHesuvQBb3nHXfynVl18979ZdZIUjHrxtVWf/oZV99L3vmzVxaJehucP9/ZZdb+72Itf3TXg1Z1ofiu0N+1FpSbczFJTysw2rTEjRofMyFJ3MyI5jnIty+3u07C5/4bGvLq55d5+fm9tuVUnSfMrvWW62+LG8GbMRDj3GDSf6iozC+PpySPVJSkV8/Z1PMfH1rtVspP+2rVHz2a/6aabcj4YAAAwffiVPQAAQsaKuNP3zjUBAICCR6cPAEBIyX6mDwBAqSnmSZ/T+wAAlAg6fQAAQuj0AQBAwaPTBwAgpJg7fSZ9AABCmPRzZPdAatKa98e92MtI1PtkouyU37bqgrrjrLrYoS6rLjN/iVUnSdX/zxetusH7PmXVPTc0ZtUdU+PF4f7mYi8Pd68RsyxJx87xHuM9g2mrrqHaOxb+88CwVTe/yntaVJR5632934tZrjDzcPtS3n5eWO1tR78Zc9s/6j0eFWbUrMuNpK2t8I4rN35akjrNY9AdYzrwlhc3x+hGKLu/GedPdt6+lrzXIvcxeU+luVocFZ/pAwAQEmSCnN+m6uWXX9bll1+uZcuW6fLLL9crr7wyoeapp57SpZdeqlNOOUVtbW3Wcpn0AQDIM62trVq+fLkef/xxLV++XOvXr59Qs3jxYn3xi1/UH/3RH9nLZdIHACAkkwlyfuvr69PevXsn3Pr6Jv6K6YEDB7Rz5061tLRIklpaWrRz5051d3ePqzvuuOPU3NyssjL/k3ou5AMAYJpt3bpV99xzz4T7V61apdWrV4+7r6OjQ42NjYrF3rp2IhaLqaGhQR0dHaqrq3tX42DSBwAgJDAvzJyKq6++WpdccsmE+5NJ70LpXGHSBwAg5J1ceDeZZDJpT/BNTU3q7OxUOp1WLBZTOp1WV1eXmpqa3vU4+EwfAIA8Ul9fr+bmZrW3t0uS2tvb1dzc/K5P7UtM+gAAjDMdF/JN1S233KIHH3xQy5Yt04MPPqgNGzZIklasWKGf/exnkqQf/ehHOvPMM3X//ffr61//us4880z94Ac/yLpcTu8DAJBnTjjhBG3btm3C/ffee+/hvz/ykY/oX//1X6e0XCZ9AABCAjfCsADN6KR/3mkLJ6055jeWWMuq/vhZVt1g7XususS/fcOqy3z0QqtuzpnXW3WSNPBPf27VRd58xapz42HN1Fwly73luTGybl1/mVdnDk8frPcO91TEq3PP2NUkzMhoM4p0boW34ljEW15ghrRWmt8FdiNV3TTcROBFuUbNiOzaOQ3eiuXH0o6ZdRVRry6ImnHk7lXm5iyWMY9989Cy999UopFnwnRcvZ8v+EwfAIASwel9AABC3smFd4WCTh8AgBJBpw8AQMh0hPPkCyZ9AABCinnS5/Q+AAAlgk4fAICQDF/ZAwAAhY5OHwCAED7TBwAABW9GO/3Xdx80ql6ylnXs5X1WXfnogFUXPfGjVl2VGa976F/vtOokScPetoye8FtWXeTQqFX3xqBXd2DIjIct96JDe60qyUzr1Qs9KatuQZV3uA+OerGvqbTXDVSascgud7+Yw7MlzBWX5Xi9w/KOK5XPt8pSQ2l73W6EsnssSN4+TI16Y3RX68bmJqJeXK+bmuuOL+GVzZhi7vQ5vQ8AQAiJfAAAoODR6QMAEMKv7AEAgIJHpw8AQEjgXc9YkJj0AQAI4UI+AABQ8Oj0AQAIKebv6dPpAwBQIrJ2+tu3b9fpp58uSTp06JA2btyoHTt2qLm5Wa2trZo/30vAetv/fmNw0pplZtTTm088btUtiHppXlXXPGzVDbbfaNWpr0Pp55+xSsd+61NWXeZ/3WHVvff8a6w6mWljkeFDVl0QqbTqYgMHrLpMxVyrrjHqpZcFQbVVV5+Z/DiVpEjGSzTMRLz1Roe8rMIg7u3nSMpLo1TEOw6ChLcdkdGh3K436p+QdI+tIBY367x1V5vboilsiyXjpUfa6831FWzm8iLudlSd+i4G4yvZTv+OO345ydx1112qrq7Wli1bdPzxx2vTpk3TPrhC5k74AHIj1xM+SlcmCHJ+yxdZ3/6FAwp+/OMf65vf/Kbi8biWLl2qCy64YNoHBwAAcifrpJ9KpbR7924FQaBIJKJ4/JfvkKNRLgcAABSfYj69n3XSHx4e1sqVKw93/J2dnWpsbFR/fz+TPgAABSbrpP/kk08e8f5YLKYvf/nL0zIgAABmUzF3+u+oXa+srNTixYtzPRYAADCNCOcBACCkmGN4mfQBAAjhp3UBAEDBo9MHACCkmC/km9FJ///9ow9PWrPkU17oT+y0c626qvNuseoGv/J7Vl2mZoFVl/qNy6w6SarsftWqG6tvspfpCGIJqy5iJphFUl78qhurGiSqvPWO9Ft1sd4Oqy5TkbTqlE55dfLia92YW2W82GE3xlhm1KwbqRrE5lh1kTFv/7mxvm7SXqa8xqp7a+XmydCIFx9uRxS7sbnu+HIcr+s+hyPuMZPreGIcFXsaAIAQLuQDAKBEBObZtELEhXwAAJQIOn0AAELo9AEAQMGj0wcAIKSYO30mfQAAQoJ08U76nN4HAKBE0OkDABBSzKf36fQBACgRM9rpd/2sa9KaIP1ta1kfWrvdqhv47i1W3RtfabPq5pvvAKuqzShXSQNPP2nVBRkv0jKRetiqi1Z5camZaMyqG/rPn1t1iXm1Vl2kvMKqC0aGrbpMyqsL0t5+LkuaMbfm/su1YNSMuY17cczufnaN9XvxyWU1Xmxu4Mb6JrzjSpLSA2bEc3m5t7yRkZwuL9fPEbfDzYyOWXWxSi9K242WLrv8z7zlvUt0+gAAoODxmT4AACHF3Okz6QMAEFLMkz6n9wEAKBF0+gAAhNDpAwCAgkenDwBASKaIO30mfQAAQji9DwAACh6dPgAAIcXc6c/opH/SZR+dtObYrx6ylvXTD79p1aV3/ItV98Mvfd+qm//IT6y6Y08/3qqTpNd/+IpVlx71DsT5J9ZbdYlkpVU3dMCLIk31j3rrrYlbdZX1Xvxqf0evVTd00IsijVd4T4vUgLe9c47xtsPl7ueaJjNm2YxUTae84y+W8GKHIzHvROPogBev6+6Xskr/ZS8W98aYHvWim0f7vW1x92GuRWKRnC7P3Y5MOrDqPjRDMbzFjE4fAICQIE2nDwBASSjm0/tcyAcAQImg0wcAIIROHwAAFDw6fQAAQuj0AQBAwaPTBwAgJMh4uQuFiEkfAIAQTu8DAICCN6OdvhOxu/f3vejQXdt2WXWRbzxq1T3eOWDVfTLjxUVKL5l10nd37LfqTqhOWHUHXvViaauS5VZdT48XX2smadqScS/Cc9Q8FfeiGdO6oNxbr5tYWrnrgFWXDrwd2J3ytndxlRd37O6//jGvLhbJbZTrUNodn7f/asr88SWiXq177A+Zhe6xlePUXFuu94t77H/IW9y7RqcPAAAKHp/pAwAQkiniTp9JHwCAkGL+wR1O7wMAUCLo9AEACOFCPgAAUPDo9AEACKHTBwCgRASZdM5vU/Xyyy/r8ssv17Jly3T55ZfrlVdemVCTTqe1YcMGnXPOOTr33HO1bdu2SZfLpA8AQJ5pbW3V8uXL9fjjj2v58uVav379hJpHH31Ue/bs0Xe/+1099NBDuvvuu7V3796sy2XSBwAgZDo6/b6+Pu3du3fCra+vb8L6Dxw4oJ07d6qlpUWS1NLSop07d6q7u3tc3WOPPabLLrtM0WhUdXV1Ouecc/TP//zPWbdtRj/TT+24L2fLarghZ4uSJP3NV3K7vKk4Y/ZWDQD4Fbmcq952991365577plw/6pVq7R69epx93V0dKixsVGx2FuR4LFYTA0NDero6FBdXd24umOOOebwv5uamrR/f/ZYdy7kAwBgml199dW65JJLJtyfTCZndBxM+gAATLNkMmlP8E1NTers7FQ6nVYsFlM6nVZXV5eampom1L3++uv64Ac/KGli538kfKYPAEAeqa+vV3Nzs9rb2yVJ7e3tam5uHndqX5LOP/98bdu2TZlMRt3d3XriiSe0bNmyrMuOBIH5m4YAAGBG7N69W+vWrVNfX5+SyaTa2tp0/PHHa8WKFbruuut06qmnKp1Oa+PGjdq+fbskacWKFbr88suzLpdJHwCAEsHpfQAASgSTPgAAJYJJHwCAEsGkDwBAiWDSBwCgRBDOMwVnnXWWEomEysvLJUk33HCDzjgj/0N029ra9Pjjj2vfvn169NFHtXTpUklv/YrTunXr1NPTo9raWrW1tWnJkiWzO9hJHG1bCumxOXjwoNasWaM9e/YokUjouOOO08aNG1VXV6ef/OQnWr9+vUZGRrRo0SL91V/9lerr62d7yEeVbVtOPPFELV26VNHoW73F7bffrhNPPHGWR3x01157rfbu3atoNKqqqirdfPPNam5uLrjnydG2o5CeI5hGAWyf/OQng+eff362hzFlzzzzTPD6669PGP9VV10VfOtb3wqCIAi+9a1vBVddddVsDdF2tG0ppMfm4MGDwQ9/+MPD/77tttuCP/uzPwvS6XRwzjnnBM8880wQBEGwefPmYN26dbM1TMvRtiUIgmDp0qVBf3//bA1tyvr6+g7//S//8i/BxRdfHARB4T1PjrYdhfQcwfTh9H4J+MhHPjIhvtH9Fad8c6RtKTS1tbX6+Mc/fvjfH/7wh/X666/rueeeU3l5uT7ykY9Ikq644opJfzFrth1tWwrRnDlzDv/d39+vSCRSkM+TI20H8DZO70/RDTfcoCAI9Ou//uu6/vrrZ/zHEnLF/RWnQlKIj00mk9E//MM/6KyzzpqQm11XV6dMJnP4tHK+C2/L26666iql02mdeeaZWr16tRKJxCyOcHJ//ud/ru3btysIAn3lK18p2OfJr27H2wrxOYLcotOfgq9+9at65JFH9PDDDysIAm3cuHG2h4T/q1Afm1tvvVVVVVW68sorZ3so79qvbsv3v/99/eM//qO++tWv6sUXX9TmzZtneYST++IXv6jvf//7+vznP6/bb799tofzjh1pOwr1OYLcYtKfgrdPKycSCS1fvlzPPvvsLI/onQv/ipOko/6KU6EoxMemra1Nr776qv76r/9a0Wj08C9mva27u1vRaLQguvxf3Rbpl49JTU2NLrvssoJ4TN528cUX6+mnn9bChQsL+nny9nYcPHiwIJ8jyD0mfdPg4KAOHTokSQqCQI899piam5tneVTvnPsrToWgEB+bO++8U88995w2b958+JT3KaecouHhYf3oRz+SJH3961/X+eefP5vDtBxpW3p7ezU8PCxJGhsb0+OPP57Xj8nAwIA6OjoO//vJJ5/U3LlzC+55crTtKC8vL7jnCKYHP7hjeu2117R69Wql02llMhmdcMIJuummm9TQ0DDbQ5vUpk2b9N3vfldvvvmm5s2bp9raWn3nO9856q845bMjbcvf/M3fFNRj88ILL6ilpUVLlixRRUWFJOnYY4/V5s2b9eyzz6q1tXXcV/bmz58/yyM+uqNtyzXXXKP169crEolobGxMp512mr7whS+ourp6lkd8ZG+++aauvfZaDQ0NKRqNau7cuVq7dq1OPvnkgnqeHG07kslkQT1HMH2Y9AEAKBGc3gcAoEQw6QMAUCKY9AEAKBFM+gAAlAgmfQAASgSTPgAAJYJJHwCAEvH/AxNGq7fPj3sQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T-aX6o6gOk3U",
        "colab_type": "text"
      },
      "source": [
        "## Cosine similarity of weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Te3tP6UPOmJr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_model_weights(model):\n",
        "  model_weights = []\n",
        "  for layer in model.layers:\n",
        "    weights = layer.get_weights()\n",
        "    if len(weights)==0:\n",
        "      continue\n",
        "    model_weights.extend(np.array(weights[0]).flatten()) \n",
        "\n",
        "  return np.array(model_weights)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U1AGZB0zPeHL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "10fe56316ccd4302be2fb49df1d4aaa9",
            "710826759a7e46f398996463b5f45df0",
            "df69fb347c8e41f0800e6419ae5e34e7",
            "594ae42dd45d4a749f4d626a1bc0c80f",
            "d80e06c932b9445dac10bf6317de56c5",
            "6e92c58f4fd048298f57b23c25e3656b",
            "f1f3f35cde634c7ca27be74b103abbff",
            "251ade832ceb4994ac0880e1e8965acf"
          ]
        },
        "outputId": "dd4389ca-fd3f-4410-b1c0-061bfaa78a05"
      },
      "source": [
        "weights_of_models = []\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model('/content/ResNet20v1_CIFAR10_Aug/' + model_ckpts[i])\n",
        "  # get predictions for model\n",
        "  weights = get_model_weights(model)\n",
        "\n",
        "  weights_of_models.append(weights)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "10fe56316ccd4302be2fb49df1d4aaa9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p3D6La8JQQ1Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "87eb3a40b6ca487dac05dead7b4ed087",
            "113b0d59cd27486391963d4cf5b10d63",
            "df3abf63839940cda915c08532a6a333",
            "7efedc14d28c4ee6b45cbc9db5446e42",
            "bcfa853efbf942e18674a01e01fb951d",
            "ceca6d5c594b4f6bb3c059136723f767",
            "ce631a54e9d442ab9e183faf099d54b4",
            "ab4f8409446b4664b9aea443011ac3d7"
          ]
        },
        "outputId": "60bb6895-dfc6-4595-ec43-7230ac508140"
      },
      "source": [
        "empty_arr = np.zeros(shape=(40, 40))\n",
        "\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  weights1 = weights_of_models[i]\n",
        "  for j in range(i, 40):\n",
        "    weights2 = weights_of_models[j]\n",
        "    \n",
        "    # compute cosine similarity of weights\n",
        "    cos_sim = np.dot(weights1, weights2)/(norm(weights1)*norm(weights2))\n",
        "    \n",
        "    empty_arr[i][j] = cos_sim\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = cos_sim\n",
        "\n",
        "cos_sim_coeff = empty_arr[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "87eb3a40b6ca487dac05dead7b4ed087",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6ZG3l94YzqT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "2f82743d-477b-4229-b281-8aaae0bd0c12"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns_plot = sns.heatmap(cos_sim_coeff, cmap='RdBu_r');\n",
        "plt.xticks([5,10,15,20,25,30,35],[5,10,15,20,25,30,35]);\n",
        "plt.yticks([5,10,15,20,25,30,35],[35,30,25,20,15,10,5]);\n",
        "plt.show()\n",
        "\n",
        "figure = sns_plot.get_figure()    \n",
        "figure.savefig('functional_similarity.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAHYCAYAAAAyFvBfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df3RV5Z3v8c85hyT8PIQgiYcfLYoFU8TWH6PXRZUrIDJzAwG7aF1Yh5m7hCoSZo1XITpKhKKLMIx2iqSuYaYOLmunCztTJSAyjqu3lUW9jiKOBlBZWBoIQUlCSCA5yTn7/sGY7jTJyTe4c35kv19rnWU4+fo8z84h5Ml37/05AcdxHAEAAF8LpnoBAAAg9dgQAAAANgQAAIANAQAAEBsCAAAgNgQAAEBsCAAASCvl5eWaOXOmpkyZoo8++qjbmlgsprVr12r27Nm67bbbtH37dtPnEhnkyeoBAIAnZs2apT//8z/XXXfd1WPNjh07dOzYMe3Zs0cNDQ1asGCBbrrpJo0fPz7h5xKhQwAAQD9rbGxUdXV1l0djY2OX2uuvv16RSCTheLt27dKiRYsUDAaVl5en2bNna/fu3b1+LpGkdgiyr/nfvdZM/bNFprFu+x9fMdXNLcw31V01ZqipblTrZ6a6wPFDpjpJin7yvqnuzCe/M9W1nO76F6z7ujO2uvoWU915Y11rY6uxLmqqi55rM9U1tcc9rTsfs4V8RuPe1sWM4aK2o7CPZzzctB+vL6xzp0p/HHM6e9b5NCnz3BuY6PmYU3/0f/TMM890eX7FihUqKSnp83g1NTUaO3Zsx58jkYhOnjzZ6+cS4ZQBAAD9bMmSJVq4cGGX58PhcApW0z02BAAA9LNwOOzpD/9IJKITJ07o6quvltS5K5Doc4lwDQEAAC6hgPcPr82dO1fbt29XPB5XXV2dXn/9dd1+++29fi4ROgQAALiEAv3wE7wP1q9frz179ujzzz/XX/7lXyo3N1c7d+7U0qVLtXLlSk2bNk3FxcU6cOCA5syZI0m6//77NWHCBElK+LlEAsl8+2MuKuweFxX2VMdFhd3hosIvN15fcFFheknWRYUrQ5d5PuaPYkc9H9NrdAgAAHDpjxZ/JuAaAgAAQIcAAAC3VF9DkCpsCAAAcOGUAQAA8K206xAEg7atWchYZxWwtoisVx3HYxe/mJ6mjtmuG3ditrkd4yXKcWOddTxrnZX9KvTUXP2eKul+tX+qvn7p/rpJ/rt7IN349ZQBHQIAAJB+HQIAAFKJawgAAIBv0SEAAMDFr9cQsCEAAMDFr61zvx43AABwoUMAAICLX08Z0CEAAAB0CAAAcPPrbYdsCAAAcPHrKYOkbggCwZChxtsXwvMX1jHGB/clutjjmGMnbltj3ByFbKvzWqrm9ZrXUbkD46tiN5BifAfSsWDgoUMAAICLX08ZcFEhAACgQwAAgBvXEAAAAE4ZAAAA/6JDAACAi19PGdAhAAAAdAgAAHDz6zUEbAgAAHDx64aAUwYAACBzOwQhY8Rx0HhxiMeJyX1ijTm2RvmmKvLXGoUcN+a3WuuscbD2Om/ntfI64thvUvn1I5J4YOGiQgAA4FsZ2yEAAKA/cA0BAADwLToEAAC4+PUaAjYEAAC4cMoAAAD4Fh0CAABc/HrKgA4BAACgQwAAgJtfryFI6oYgEAwZamyvhDWpMGTsgVhf/4BjTAEcXyjn2IemUiduTCA01lk5cWNioLHOOp7Xx2FPFkxNnFyqEg1TlcxI4iIyHacM4CnrZgAAgHTAKQMAAFys74Ez0NAhAAAAdAgAAHAL+PSqQjYEAAC4BH26IeCUAQAAoEMAAIBbwHq/+gDjz6MGAACd0CEAAMDFrxcV0iEAAACJOwR79+7V9OnTJUlnz57VunXrtH//fhUWFqqsrEyXXHJJUhaZDMYkZMkYXezEYxe/mJ7GjBnnNtbFjVm0jtfZu0apmjdVEb3eBjqnvxS9vH2SCWuE97jLoBubNm3q+Pjpp5/WsGHDVFFRocsvv1zr16/v98UBAJBsgWDQ80cmSNghcFy/Ab3zzjt66aWXlJWVpcmTJ2vevHn9vjgAAJAcCTcE0WhUR44ckeM4CgQCysrK6vhcMEN2PAAA9IVfTxkk3BC0tLRo6dKlHX+ura1VQUGBmpqa2BAAADCAJNwQvPHGG12eO3PmjHJycvSjH/2o3xYFAECqcNthNw4dOqQ77rhDixYt0pEjR7Rs2TLdcsstuv3229Xc3JysNQIAkDSBUNDzRyZIuMr169fr/vvv11133aV77rlHRUVFOnDggMrKyrRhw4ZkrREAAPSzhBuC5uZmzZo1SwsWLJAkzZ8/X5I0c+ZMNTQ09P/qAABIsmAo4PkjEyTcELhvO/wioOgL8bjfYlQAABi4El5UOG7cODU1NWn48OGdgohOnjypIUOG9PviAABItoA5unZgSbgh2LJlS7fPh8NhVVRU9HmyQDDUa00wYHshQsYXzDpewFhnjS5WH6KLvY8kTk33xhqFbK4zRgPbo4ZtdV7zOuLYb7z++hFHnH7SraMezJCLAL12Ue92OHToUA0dOtTrtQAAgBTh7Y8BAHAhhwAAAPgWHQIAAFzoEAAAAN+iQwAAgAt3GQAAAE4ZAAAA/6JDAACAS9CnSYV0CAAAQPp1CKwZ0tbo4pA1kthrfXjzJ2sksXk849xeRyHbI5jTOzvWGpXr9WF4PW+qxkuVNP9r5UuZeio+wEWFAAAgU96u2Gv+3AYBAIBO6BAAAODi19sO2RAAAJBmjh49qtLSUjU0NCg3N1fl5eWaOHFip5rPPvtMa9asUXV1tdrb23XvvfequLhYkrR582a9+OKLys/PlyRde+21KisrSzgnGwIAAFzS4aLCsrIyLV68WMXFxXr55Ze1Zs0aPf/8851qNmzYoKuuuko//vGPVVdXpzvuuEM33HCDIpGIJGnBggVavXq1ec7UHzUAAGkkGAp4/mhsbFR1dXWXR2NjY5f5T58+raqqKhUVFUmSioqKVFVVpbq6uk51hw4d0s033yxJysvL05VXXqlXX331oo+bDgEAAP1s27ZteuaZZ7o8v2LFCpWUlHR6rqamRgUFBQqFQpKkUCik/Px81dTUKC8vr6Nu6tSp2rVrl6ZNm6bq6mrt379f48eP7/j8zp079eabb2rMmDEqKSnRNddck3CNbAgAAHCx5uH0xZIlS7Rw4cIuz4fD4Yses7S0VE8++aSKi4s1duxY3XTTTR2biDvvvFP33nuvsrKytHfvXi1fvly7du3SqFGjehyPDQEAAP0sHA6bf/hHIhHV1tYqFospFAopFovp1KlTHdcGfCEvL0+bNm3q+PPSpUt1xRVXSJLGjBnT8fz06dMViUT08ccf64YbbuhxXq4hAADAJRgKev7oi9GjR6uwsFCVlZWSpMrKShUWFnY6XSBJ9fX1am9vlyTt27dPH330Ucd1B7W1tR11Bw8e1PHjx3XZZZclnDepHYJAMJTM6SRJQWN0sblD5HgbM3xhSGOEcB/ikE3zGrNeUxU1bI/eTU3UsJX3f2PSGxHCA59Pb9NPqscff1ylpaWqqKhQOBxWeXm5pAtdgJUrV2ratGl6//339cQTTygYDGrUqFF69tlnNWTIEEnSU089pQ8//FDBYFBZWVnauHFjp65BdzhlAACASzoEE02aNEnbt2/v8vzWrVs7Pp4xY4ZmzJjR7f//xQaiL9gQAADgkg45BKngz6MGAACd0CEAAMAlEPTn78r+PGoAANAJHQIAAFz6epvgQMGGAAAAFy4qBAAAvkWHAAAAFzoEAADAt9KuQ2B9l6mQx+9GZR0tYIwudmIx89xOzDqmt3Vxj6OL7XXpHeZrjd61Riaje3z9kicNgvcyil9vO0y7DQEAAKkUCCX/fXfSgT+3QQAAoBM6BAAAuHBRIQAA8C06BAAAuAR9elGhP48aAAB0QocAAAAXv15DwIYAAAAXv24I/HnUAACgE1OHoL6+XidPnpQkXXrppRo1alS/LgoAgFQhqbAbx44d02OPPaaqqirl5+dLkk6dOqWvf/3rWrt2rSZOnNinySzpT0GPo4utnZ9gwJjtGTfG7sbt0cVW1jhka4Sw16xRyFb2CGFrXXpH5Xp9HH77uvjRQIkkDln//UW/SrghWLVqlRYvXqznnnuu4zaMeDyuHTt2aPXq1fr5z3+elEUCAJAsXEPQjYaGBs2fP7/TPZnBYFDFxcU6c+ZMvy8OAIBkC4SCnj8yQcJV5ubmqrKyUo6rpeg4jl555RWFw+F+XxwAAEiOhKcMNmzYoLKyMq1bt04FBQVyHEe1tbUqLCzUhg0bkrVGAACSJpghv9F7LeGGYOLEidq2bZvq6upUU1MjSYpEIsrLy0vK4gAAQHIk3AYdOnRId9xxh5YtW6bBgwfr7//+73XrrbdqxowZOnjwYLLWCABA0gSCQc8fmSDhKtevX6/7779f3/ve93TPPfeoqKhIBw4cUFlZmcrLy5O1RgAAkoaLCrvR3NysWbNmacGCBZKk+fPnS5JmzpyphoaG/l8dAABIioTXELjvLpg+fXqnz8WtAT0AAGSQTPmN3msJj3rcuHFqamqSdOH0wRdOnjypIUOG9O/KAABA0iTsEGzZsqXb58PhsCoqKvo8WTDYe3SxlTW62BxJbGWNee1DdLET87bbEjeOZ53XOp6V1xHH6S7do4GtfPaypVS6RxIP9KjhTLkI0GsX9fbHQ4cO1dChQ71eCwAASJGL2hAAADBQBQ1vxDcQsSEAAMCFiwoBAIBv0SEAAMCFDgEAAPAtOgQAALhw2yEAAOCUAQAA8K+kdggChqTCoDGiy5pUaA3UCsoYw+YYUwDb22zjSYq12Wrjbe22ueO2Y4m1GY/FPJ4tndExRt5FjfNa66xJe9bxrPmN1nmtiYapGs8qVfP2BxIDv5x0//r1hA4BAADwLa4hAADAhYsKAQCA6fT2QOTPbRAAAOiEDgEAAG50CAAAgF/RIQAAwM2nFxX686gBAEAndAgAAHAJhPx5DQEbAgAA3Hx6UWFSNwTBQVm91xgjibON0ZLWaM9A3BYLbK1T3BbjK0nxqG3MmLnONnc8ZgvftY5njSRuixvn9TgC1x5xnKo6UxmRxD1IZUwuEcJfTrp//fyCDgEAAG4+7RBwUSEAAKBDAACAG+9lAAAAOGUAAAD8iw4BAABudAgAAIBf0SEAAMCFiwoBAACnDAAAgH8ltUMQMOy6gsZI4ixjxLFxOHvUcKzNVOa02+ouDGmLJI4b62JRWzRw3FhnHc8acWyP6LXWeRshbDva1B2H1wZKJHEmxN8SIdzTvCmZtmd0CAAAgF9xDQEAAC5+fftjOgQAAIAOAQAAnXDbIQAA4KJCAADgW3QIAABwsdwiPxDRIQAAAHQIAADoxKcXFfrzqAEA6EEgGPL80VdHjx7Vd7/7Xd1+++367ne/q08//bRLzWeffab77rtP8+bN05/+6Z/q5Zdf7vhcLBbT2rVrNXv2bN12223avn17r3Mm7BB8/PHH+trXviZJamtrU0VFhd59910VFhbqr/7qrzRkyJA+HWBwUHavNaFBxuhiYyaxNYozELfFAlsjiZ22qKlOkuJRaySxLRrYiRmjhtuM48Vt4znGbNto3Nu6dI9C9pr3x/ElFvMlpCqSOO1icrvhtwjhTIidTraysjItXrxYxcXFevnll7VmzRo9//zznWo2bNigq666Sj/+8Y9VV1enO+64QzfccIMikYh27NihY8eOac+ePWpoaNCCBQt00003afz48T3OmfCn6qpVqzo+3rx5sw4fPqy/+Iu/UENDg5588skvebgAAKShYMjzR2Njo6qrq7s8Ghsbu0x/+vRpVVVVqaioSJJUVFSkqqoq1dXVdao7dOiQbr75ZklSXl6errzySr366quSpF27dmnRokUKBoPKy8vT7NmztXv37oSHnbBD4Lh+w/jNb36jF154QcOGDdPNN9+sBQsWGL6qAABg27ZteuaZZ7o8v2LFCpWUlHR6rqamRgUFBQr9d4RyKBRSfn6+ampqlJeX11E3depU7dq1S9OmTVN1dbX279/f0QGoqanR2LFjO2ojkYhOnjyZcI29bghaWlrkOI5CoZCGDRt24X8aNEiDBnE9IgBgAOqHiwqXLFmihQsXdnk+HA5f9JilpaV68sknVVxcrLFjx+qmm27q2ERcjIQ/1Q8fPqxrrrlGjuMoEAiotrZWBQUFam1tVdx4XhkAgEzSH29uFA6HzT/8I5GIamtrFYvFFAqFFIvFdOrUKUUikU51eXl52rRpU8efly5dqiuuuKJjjBMnTujqq6+W1LVj0J2E26BDhw7p4MGDHf8tKCiQJLW2tmrdunWmAwMAAHajR49WYWGhKisrJUmVlZUqLCzsdLpAkurr69XefuGi9H379umjjz7quO5g7ty52r59u+LxuOrq6vT666/r9ttvTzhvwg5BfX29/u7v/k4nTpzQrFmzdNddd0m6sNP5p3/6J23evPnijhYAgHSVBkmFjz/+uEpLS1VRUaFwOKzy8nJJF7oAK1eu1LRp0/T+++/riSeeUDAY1KhRo/Tss8923P1XXFysAwcOaM6cOZKk+++/XxMmTEg4Z8INQVlZmcaPH69bbrlFP/vZz7Rv3z798Ic/1KBBg1RdXe3FMQMAgD8yadKkbrMDtm7d2vHxjBkzNGPGjG7//1AopLVr1/ZpzoSnDD799FOtWrVKc+bM0U9+8hONGTNG3//+99Xa2trpDgQAAAaMfrjtMBMk3BC0tf0hhCcQCKisrEyTJ0/WsmXL1Nra2u+LAwAg2QLBoOePTJBwlRMmTNDbb7/d6bnVq1frG9/4RrcxigAAIDMlvIZg48aNCnQTKfnAAw9o/vz5fZ7McitHtjG6OGiMujRHcVqji62RxMaIY6kP0cDWKN8223hxY2ZtLGobr814HF5H6nodNZy6KOTUjOe1gRRJTIRwsuZNybQ9y5AWv9cSbghyc3N7/NwX9zoCAIDMR9wgAABugcw45+81fx41AADohA4BAABuPu0QsCEAAMDF8emGwJ9HDQAAOqFDAACAGx0CAADgV3QIAABwS1FAU6qxIQAAwC1D3nvAa0ndEAQHZfdaEwradmZZxqxL63iK2aKLrZHE8T5EF8ejtrlj0ZindXFjnROzRhKbyjyP6I0aI51tR5HKKGQiidNp3r7M7f28/ooQTtXxojM6BAAAuHDbIQAA8C06BAAAuPm0Q8CGAAAAN59uCPx51AAAoBM6BAAAuNEhAAAAfkWHAAAAF247BAAAvkWHAAAAN592CJIbXRwM9VqTPcj2QmQZI4mtkZiBuC0+2LFGFxvjiCUp1maNLjZGCFsjjtusUcjW6GJvo4ZTFYWc7vN6zetoYK/jb1M1b1/m9n7elEzr+fFm7I9Vn0YpZ+zrBQAAvMMpAwAA3Hx6ysCfRw0AADqhQwAAgItfbztkQwAAgFvQnxsCfx41AADohA4BAABuPj1l4M+jBgAAndAhAADAzacdguQmFQ7K7rVm32vv6n/+r+t7rfvF/uO68/oJvda9U3NWN4wb0Wvd6ZwxGn3+ZK91+uo0OUfe6bVs8OSrdO7D93ofT1K8zZZ+aE0WdIxJgHFjNF48ZQmE1vFSkwSY7gmEkv9SCL1O2iMxsHupSnBMGp9uCNLuqC2bAUmmzYAk02ZAkm0zIJk2A5LMmwGgv7AZANAXnDIAAMDFrzkE/jxqAADQCR0CAADc6BAAAAC/okMAAICbTy9QZUMAAIAbpwwAAIBf0SEAAMCF2w4BAIBvmTsEjY2NkqRwOHzRk1mii7MHhUxjZQWtKWe2ukC83VRnjhmO2uokKR61zR2L2qKLrXVxY13qIolt48VtZWbpHkmcqgTCVMUC90eSYqpSDdM9QjhVaZRpx6cdgoQbgrq6Om3atEmvvvqqJMlxHAWDQc2dO1cPPvig8vLykrJIAPBKxv6QQtI4Pv07knAb9NBDD2nChAl64403tH//fr333nt6/fXXNX78eD300EPJWiMAAOhnCTcEx48f13333adRo0Z1PJeXl6fly5erurq63xcHAECyOY73j0yQcEOQk5Oj/fv3d3n+3XffVXZ279cDAACAzJDwGoK1a9dq1apVysnJ0bhx4yRd6Bq0traqvLw8KQsEACCZ4pnyK73HEm4IvvnNb+q1117TBx98oJqaGklSJBLRVVddpYBPL7oAAAxs/twO9HLKoL6+Xo899piefvppnTp1SnPmzNG0adMUCARUUlKSrDUCAIB+lnBDUFZWpnA4rDvvvFP/8R//oRUrVqi9/cI987///e+TskAAAJIp7nj/yAQJNwSffvqpVq1apTlz5ugnP/mJxowZo+9///tqbW1N1voAAEASJNwQtLlS+QKBgMrKyjR58mQtW7aMTQEAYEByHMfzRyZIeFHhhAkT9Pbbb+tP/uRPOp5bvXq1nnrqKW3durXPkwWCvccSh4yRxFkhW7SkOYrTGF2s9qhtuDbjeJJixlrHmJVrrYsb6+xRvqmJOLYiknhgj5fKsFm/RQin+/pwcRJuCDZu3Njt3QQPPPCA5s+f32+LAgAgVTLlnL/XEm4IcnNze/zcFVdc4fliAABINZ/uB3j7YwAA0Ie3PwYAwA/8esqADgEAAKBDAACAW6bcJug1NgQAALjEU72AFOGUAQAAoEMAAICbT88Y0CEAAABJ7hBkDxvRa83IIVmmsYZn9x6DLEmDB9n2PMHoeVNd27mzprpo4zlTnSS1NdvikKPNbb0X9aHufDRmqzNm+drrbGfoWoz3/ljHs64vapw32xiz7XVEr9VAiRq2jmd9PfpDukf5+m19X5ZfbzvklAEAAC5+vcuAUwYAAIAOAQAAbtx2CAAAfIsOAQAALulwCcHRo0dVWlqqhoYG5ebmqry8XBMnTuxUc/r0aT388MOqqalRe3u7brzxRj366KMaNGiQNm/erBdffFH5+fmSpGuvvVZlZWUJ52RDAABAmikrK9PixYtVXFysl19+WWvWrNHzzz/fqebZZ5/VpEmT9A//8A9qa2vT4sWLtWfPHv3Zn/2ZJGnBggVavXq1eU42BAAAuMT7oUXQ2NioxsbGLs+Hw2GFw+FOz50+fVpVVVV67rnnJElFRUX6wQ9+oLq6OuXl5XXUBQIBNTc3Kx6PKxqNqq2tTQUFBRe9RjYEAAC49McZg23btumZZ57p8vyKFStUUlLS6bmamhoVFBQoFLqQtxMKhZSfn6+amppOG4Lly5erpKRE3/rWt3T+/Hnddddduu666zo+v3PnTr355psaM2aMSkpKdM011yRcIxsCAAD62ZIlS7Rw4cIuz/9xd6Avdu/erSlTpmjbtm1qbm7W0qVLtXv3bs2dO1d33nmn7r33XmVlZWnv3r1avny5du3apVGjRvU4HncZAADgEne8f4TDYY0fP77Lo7sNQSQSUW1trWKxC2mysVhMp06dUiQS6VT3wgsvaP78+QoGgxoxYoRmzpypt956S5I0ZswYZWVdSP6dPn26IpGIPv7444THndzoYkMsce5QW3TxMGN0cSjWaqpzWppMdfHmrueAutN2rsVUJ0mtZ2xrbG201UXPGaOLjZG/Te3eRgMPlEjiIca81XSPJPY6Qtjr9XkdEd0X6R7Rm6pI4nSPQs50o0ePVmFhoSorK1VcXKzKykoVFhZ2Ol0gSePHj9evf/1rXX311YpGo9q3b59uu+02SVJtbW3H9QQHDx7U8ePHddlllyWcl1MGAAC4pMNth48//rhKS0tVUVGhcDis8vJySdLSpUu1cuVKTZs2TY888ojKyso0b948xWIx3XjjjfrOd74jSXrqqaf04YcfKhgMKisrSxs3btSYMWMSzsmGAAAAl3i/XFbYN5MmTdL27du7PL9169aOj7/yla903Inwx77YQPQF1xAAAAA6BAAAuKXDKYNUoEMAAADoEAAA4Ga8IWnAYUMAAIALpwwAAIBv0SEAAMAlHW47TAU6BAAAILkdgpwhvU+XNzzbNNaQQba9TKCt2VRnjiQ+e85W12ibV5Kizbao4bYmW501arip3RohbKtrNkcN2+qsUcOpiiQeEvJ2P52qCGF7NHBqoou9Po6+SPeIXqKL+wfXEAAAAN/iGgIAAFziPm0RsCEAAMDFeFZzwOGUAQAAoEMAAICbX08Z0CEAAAB0CAAAcIv5tEPAhgAAABdOGQAAAN+iQwAAgItfbztM6oZg8NDeY4lHZNuWNNiYnRloNUYXn20w1bU2nLXVNZ431V2obTXVNbfHTHX26GJvo4a9jiS2RiZbo4bt0cW2xtngARL5m7ooZNt41vX5Mbo4aBwv4PGBBK3//noc743+lfDV2rt3b8fHZ8+e1UMPPaTZs2erpKREn3/+eb8vDgCAZIs7juePTJBwQ7Bp06aOj59++mkNGzZMFRUVuvzyy7V+/fp+XxwAAMkWcxzPH5kgYX/ecR3EO++8o5deeklZWVmaPHmy5s2b1++LAwAAyZFwQxCNRnXkyBE5jqNAIKCsrKyOzwWDnBsCAAw8xsucBpyEG4KWlhYtW7aso1NQW1urgoICNTU1sSEAAGAASbgheOONN7p9PhQKafPmzf2yIAAAUinm0xbBRf2aP2TIEN13331erwUAAKRIwg7BJ5980uPn6uvrPV8MAACplim3CXot4YagqKhI48aN63S3wRcaGmxBPgAAZBJjJtqAk3BDMG7cOL344osqKCjo8rkZM2b026IAAEByJdwQzJkzR8ePH+92Q3Dbbbf1ebKcIb3HEodzbNHFQ7Jslz8EzzaZ6trP2SKJo43nTHWtjVFTnSRFm9tMdamKJG4xXmDjt0hi63hECH+5ebOMdzRZ43mtsbt94XVEb6qOJWD8Wns+bz+8Jl8Gpwy6sXr16h4/9+ijj3q+GAAAkBq82yEAAC5+ve2QDQEAAC5+PWVA3CAAAKBDAACAm19vO6RDAAAA6BAAAODm12sI2BAAAOAS9+ldBpwyAAAAye0QDB6S1WvNyMHGpMJBxvSyVltSYfys7c2a8qZeppO//aDXutzLR6v2vROmMVvP2lINm9ptu1ZrUqE1gdCaaJjuCYTDjHXW9Y0ad1YAABUTSURBVFnrUpdAmN6JgaHskKfjSVIoyzamlTWRz+ukPWvyofdJhdaExNQkMyYLFxXCxLIZkGTeDADwhtebAcBvuIYAAAAXv15USIcAAADQIQAAwC3m0w4BGwIAAFy47RAAAPgWHQIAAFy47RAAAPgWHQIAAFz8etshGwIAAFy4yyAJRg/P7rVmZE7v8caSFGpvMdXFm8+a6qINtojjaOM5U11rY6upTrJHDQ+USGJr1PBgj6OLresbbozF9jq62BpJHLRGEmdbo4ZtdebxjBHH1vHMEcfGr5+Uuuhda12q1hfsw9fQNm9qIphxcegQAPAVr3/YYuCJcdshAADwKzoEAAC4+LVDwIYAAAAXv24IOGUAAADoEAAA4EaHAAAA+BYdAgAAXPzaIWBDAACAi183BJwyAAAAye0Q5A3L6bVmmDUetbXRVBdvajDVRc82m+pa6m2Rya2NUVOdZI8kbjZHDfsrktgaNTx8kMdRyMZIXWssa9Dj8UJZxvFSNq811tYYcZzlbVRzX1i/NvaIY2/XaF6fx9HF1tfO+nVJFjoEAADAt7iGAAAAFzoEAADAt+gQAADg4tcOARsCAABc/Loh4JQBAACgQwAAgFs7HQIAAOBXdAgAAHDx6zUEbAgAAHBhQ5BAfX29Tp48KUm69NJLNWrUqIuabPTw7F5rhhhjaIOttqjh9rP1prrW+iZTXUtjq6kueq7NVCfZo4sHSiSxPWo4NXXZQ7NsdcNsddZY1pAxQtgaL2sdz1zncTRwyBhd7PXXxRqnK6UuajgQ9DbW2fP1Wec1HkcgZHuN0b8SbgiOHTumxx57TFVVVcrPz5cknTp1Sl//+te1du1aTZw4MRlrBAAgaWJO6jsER48eVWlpqRoaGpSbm6vy8vIuP3NPnz6thx9+WDU1NWpvb9eNN96oRx99VIMGDVIsFtP69ev1m9/8RoFAQMuWLdOiRYsSzplw+7Zq1Sp9+9vf1ltvvaWdO3dq586deuutt3THHXdo9erVX/qAAQBAV2VlZVq8eLFee+01LV68WGvWrOlS8+yzz2rSpEnasWOHXnnlFX344Yfas2ePJGnHjh06duyY9uzZo5///OfavHmzqqurE86ZcEPQ0NCg+fPnK+hq+wSDQRUXF+vMmTMXc4wAAKS1WNzx/NHY2Kjq6uouj8bGru/ce/r0aVVVVamoqEiSVFRUpKqqKtXV1XWqCwQCam5uVjweVzQaVVtbmwoKCiRJu3bt0qJFixQMBpWXl6fZs2dr9+7dCY874YYgNzdXlZWVclztE8dx9MorrygcDtu+sgAAZJD+2BBs27ZNs2bN6vLYtm1bl/lrampUUFCg0H9fWxEKhZSfn6+amppOdcuXL9fRo0f1rW99q+Nx3XXXdYwxduzYjtpIJNJxLWBPEl5DsGHDBpWVlWndunUdu47a2lpdeeWV2rBhg+HLCgAAlixZooULF3Z5/sv8cr17925NmTJF27ZtU3Nzs5YuXardu3dr7ty5FzVewg3BxIkTtW3bNtXV1XXsTCKRiPLy8i5qMgAA0l1/3HYYDofNP/wjkYhqa2sVi8UUCoUUi8V06tQpRSKRTnUvvPCCnnzySQWDQY0YMUIzZ87UW2+9pblz5yoSiejEiRO6+uqrJXXtGHTHdE9IXl6epk6dqqlTp3ZsBubNm2c6MAAAYDd69GgVFhaqsrJSklRZWanCwsIuv4yPHz9ev/71ryVJ0WhU+/bt09e+9jVJ0ty5c7V9+3bF43HV1dXp9ddf1+23355w3oQdgk8++aTb5x3HUX297f5+AAAySSxuy3zpT48//rhKS0tVUVGhcDis8vJySdLSpUu1cuVKTZs2TY888ojKyso0b948xWIx3XjjjfrOd74jSSouLtaBAwc0Z84cSdL999+vCRMmJJwz4YagqKhI48aN63RR4RcaGhou6iABAEBikyZN0vbt27s8v3Xr1o6Pv/KVr+i5557r9v8PhUJau3Ztn+ZMuCEYN26cXnzxxY4LCt1mzJjRp4kAAMgERBd3Y86cOTp+/Hi3G4Lbbrutz5ONNETC5gwyRoU22qKL42dtnYzoWdt4bc22SGJrzPCFWttfvlRFEg8z1tkjhL2NOB42yBZ7mjXcFjWcE84x1aUqujiUZY0kTk00sHW8YJbtrVRC2bY663h9YY3UtUb5Bo1RvlbmCGHPo4ZTcxzJwoagG4nSCB999FHPFwMAAFKDdzsEAMCl3acdgvTq0wAAgJSgQwAAgAvXEAAAAN9uCDhlAAAA6BAAAOBGhwAAAPgWHQIAAFz82iFgQwAAgAsbgiS4+tLe3ws6z7FFCLf/7qCprv7Qp6a604c/s9X97oyp7qgx4liSfn/eVjthiC0q1xpdbI8a9rYuZ0S2qc4aDex11HDOSNt4WcNsxxEyRuoGzRG9tuPwel5rrK11PK/XFzJ+XfojJjdV0cCeRwhb1xe0RTrLWoe0QIcAAAAXx6cdAi4qBAAAdAgAAHCL0yEAAAB+RYcAAAAXx/Fnh4ANAQAALlxUCAAAfIsOAQAALlxUCAAAfIsOAQAALk481StIjaRuCCaPHtJrzaBTB0xjnTn4X6a6z97/1FT3+aHTprpPmmwxw5+es0cXTxxqi1y9zBi9O8QYZ5ptnDcnbIvotUYI2+us8/b+90qSssLDbHVDB5vqssNDTXVBY0RvKNv2eljHM0chD7LNK2NdIMv2ugWs8xrjb70er09zhzyO6DVGCJvXZ44a9joKOTMjjv16lwGnDAAAAKcMAABw46JCAADgW3QIAABw8WswERsCAABc/Loh4JQBAACgQwAAgFuc2w4BAIBf0SEAAMCFawgAAIBvJbVDcGnwXK81Lf+1zzRW7TuHTHUn3ztlqjvc0GKqO9IcNdVNGmaLb5WkK4bbIldHf3WkqS7LGHE82BghPHiUt1G+2SNsEcLW8XJyR5jqskYYo4aHhW11Q23zapAxyjfLGA1sHS/dI4lDtjonaPxnKmD8/SYQsNVJ5ihfxzy3x7+DeTyv+TiM7F+XPrwmSeDXDgGnDAAAcCGpEAAA+BYdAgAAXHi3QwAA4Ft0CAAAcHHiqV5BarAhAADAhYsKAQCAb9EhAADAxa85BHQIAABA4g7B3r17NX36dEnS2bNntW7dOu3fv1+FhYUqKyvTJZdc0qfJgr97r9ea2n0HTGOd+H8nTHWHP+s9HVGSPmqyJRBOHm5PIJySa0v4u+TK0aa60VPGmOqsSYA5o4Z7Ol52rm284IhRtjpjEmBwRK6tzphAGBhsO4549hBTnYxJe+ZEvmDIVBa3jheyzpua4zCPJykubxPvrO96Z/190utfPK23x1nnTffjyPd22h7RIejGpk2bOj5++umnNWzYMFVUVOjyyy/X+vXr+31xmcy6GQDgDa83A/CvuON4/sgECbfe7l3bO++8o5deeklZWVmaPHmy5s2b1++LAwAAyZFwQxCNRnXkyBE5jqNAIKAs15uvBI1v+gEAQCbx6ymDhBuClpYWLVu2rKNTUFtbq4KCAjU1NbEhAABgAEm4IXjjjTe6fT4UCulHP/pRvywIAIBU8muH4KJ+zR8yZIgmTJjg9VoAAECKEEwEAICLX6OL2RAAAODC2x8DAADfokMAAICLXy8qTOqGoOmtX/Va8/vfHDGNVfW7M6a6g2dbTXWFI3JMdVPGDDXVXfpNe8jmmKsnmupGXWmrM0f5WiOEh1ujgW1Rw06OMRrYWOfk2KKV27NsUcOxkO3vQku77U3TY8Z/W2LGNmXM+F7t5vGMC4y1ezyv8R/dmGOLFbd2efuSGmf9WltZ57Z+Db1OwLMer9fHYTVzpO17HReHDgEAAC5cVAgAAOTEY6leQkpwUSEAAKBDAACAGx0CAADgW3QIAABw8WuHgA0BAAAuTsyfGwJOGQAAADoEAAC4+fWUAR0CAACQ3A7Bp6+902vNh4frTGN90GiLJL4qbIuh/fpXR5rqxt4w1lRXcN2VpjpJGlo4zVQ36KuFpjprlG8821YXM0YIRwcNNtWdN0b+nm+z1bW021LFzrfYxmuOnjPVNUVtv0W0GfNg24zpaN6P521MrtfrixrHM0ch9yGFrj/GZN6LH2/mFWM8nbcndAgAAIBvcQ0BAAAufu0QsCEAAMDFrxsCThkAAAA6BAAAuNEhAAAAvkWHAAAAl3gadAiOHj2q0tJSNTQ0KDc3V+Xl5Zo4cWKnmlWrVunw4cMdfz58+LC2bNmiWbNmafPmzXrxxReVn58vSbr22mtVVlaWcE42BAAAuKTDKYOysjItXrxYxcXFevnll7VmzRo9//zznWo2btzY8fGhQ4e0ZMkS3XzzzR3PLViwQKtXrzbPySkDAAD6WWNjo6qrq7s8Ghsbu9SePn1aVVVVKioqkiQVFRWpqqpKdXU9B/e99NJLmjdvnrKzsy96jXQIAABw6Y8OwbZt2/TMM890eX7FihUqKSnp9FxNTY0KCgoUCoUkSaFQSPn5+aqpqVFeXl6XMaLRqHbs2KF//ud/7vT8zp079eabb2rMmDEqKSnRNddck3CNSd0Q/Nf/PdZrzYEzLaaxvjHSFpM7dUrXL153Jtw8yVRXcNM3THWDp91kqpOk9vyvmeo+C9iihlutUb7GCOHmBlvdmdaztrqWdlNdY6ut7mzUVlfXFDXVNZxrM9WdOW+rs8ayRttt/wi1Gl+3qLHOvj7jeMa6uDGSOG5cX9wYwdwX1rkdj6N8reNZ46S9ntfr8axfZ82b+iVWk1pLlizRwoULuzwfDoe/9Nivv/66xo4dq8LCP8Tb33nnnbr33nuVlZWlvXv3avny5dq1a5dGjRrV4zh0CAAAcHFi3ncIwuGw+Yd/JBJRbW2tYrGYQqGQYrGYTp06pUgk0m39L37xC33729/u9NyYMX9434fp06crEono448/1g033NDjvFxDAACAixOPef7oi9GjR6uwsFCVlZWSpMrKShUWFnZ7uuDkyZN65513NG/evE7P19bWdnx88OBBHT9+XJdddlnCeekQAACQZh5//HGVlpaqoqJC4XBY5eXlkqSlS5dq5cqVmjbtwrvk/tu//ZtuvfVWjRzZ+R17n3rqKX344YcKBoPKysrSxo0bO3UNusOGAAAAl3S47XDSpEnavn17l+e3bt3a6c/33Xdft///FxuIvuCUAQAAoEMAAIBbOnQIUoEOAQAAoEMAAICbE7dlZAw0bAgAAHDhlAEAAPCtpHYI9p4+32vN9NFDTGNNm/EVU93E268z1Q2/8X+a6uJf/aaprjo+1FQnSR991vvXRZLeP3nSVHfGGL172hjlW9fc6ul4LcbI39bztkjilnO2ea3jRY3rizbbopqtv23E223HYa7zeF5rept5PPP6bK+Hebw+/PbndWKd1795+m08PfGn3o7XAzoEAADAt7iGAAAAl750kQYSNgQAALj0x5sbZQJOGQAAADoEAAC4cVEhAADwLToEAAC4+LVDwIYAAAAXv24IOGUAAADoEAAA4ObXDkHAcRwn1YsAAACpxSkDAADAhgAAALAhAAAAYkMAAADEhgAAAIgNAQAAEBsCAAAgNgQAAEBsCAAAgNgQAAAA8V4GfTJz5kxlZ2crJydHkvTggw/q5ptvTvGqeldeXq7XXntNx48f144dOzR58mRJ0tGjR1VaWqqGhgbl5uaqvLxcEydOTO1ie9HTsWTSa1NfX69Vq1bp2LFjys7O1le/+lWtW7dOeXl5eu+997RmzRq1trZq3Lhx+tu//VuNHj061UvuUaJjmTJliiZPnqxg8MLvHRs3btSUKVNSvOKeLV++XNXV1QoGgxo6dKgee+wxFRYWZtz3SU/HkUnfI0gRB2a33nqrc/jw4VQvo8/efvtt58SJE13Wf/fddzu//OUvHcdxnF/+8pfO3XffnaolmvV0LJn02tTX1zu//e1vO/68YcMG5+GHH3ZisZgze/Zs5+2333Ycx3G2bNnilJaWpmqZJj0di+M4zuTJk52mpqZULa3PGhsbOz7+93//d2fBggWO42Te90lPx5FJ3yNIDU4Z+MD111+vSCTS6bnTp0+rqqpKRUVFkqSioiJVVVWprq4uFUs06+5YMk1ubq5uvPHGjj9/85vf1IkTJ/TBBx8oJydH119/vSTpzjvv1O7du1O1TJOejiUTjRgxouPjpqYmBQKBjPw+6e44AAtOGfTRgw8+KMdxdN111+mBBx5QOBxO9ZIuSk1NjQoKChQKhSRJoVBI+fn5qqmpUV5eXopXd3Ey8bWJx+P62c9+ppkzZ6qmpkZjx47t+FxeXp7i8XhHqzrduY/lC3fffbdisZhuueUWlZSUKDs7O4Ur7N3f/M3faO/evXIcR//4j/+Ysd8nf3wcX8jE7xEkDx2CPvjpT3+qV155Rb/4xS/kOI7WrVuX6iXhv2Xqa/ODH/xAQ4cO1fe+971UL+VL++Nj+dWvfqV//dd/1U9/+lN98skn2rJlS4pX2LsnnnhCv/rVr/TXf/3X2rhxY6qXc9G6O45M/R5B8rAh6IMvWtXZ2dlavHix3n333RSv6OJFIhHV1tYqFotJkmKxmE6dOpWx7fhMfG3Ky8v1u9/9Tj/84Q8VDAYViUQ6tdvr6uoUDAYzojvwx8ci/eE1GT58uBYtWpQRr8kXFixYoLfeekuXXnppRn+ffHEc9fX1Gfk9guRiQ2B07tw5nT17VpLkOI527dqlwsLCFK/q4o0ePVqFhYWqrKyUJFVWVqqwsDCt26A9ycTX5qmnntIHH3ygLVu2dLTRr7rqKrW0tOg///M/JUn/8i//orlz56ZymSbdHcuZM2fU0tIiSWpvb9drr72W1q9Jc3OzampqOv78xhtvaOTIkRn3fdLTceTk5GTc9wiSL+A4jpPqRWSC3//+9yopKVEsFlM8HtekSZP06KOPKj8/P9VL69X69eu1Z88eff755xo1apRyc3O1c+dOHTlyRKWlpWpsbFQ4HFZ5ebkuv/zyVC83oe6O5dlnn82o1+bjjz9WUVGRJk6cqMGDB0uSxo8fry1btujdd99VWVlZp9sOL7nkkhSvuGc9Hcs999yjNWvWKBAIqL29Xddcc40eeeQRDRs2LMUr7t7nn3+u5cuX6/z58woGgxo5cqRWr16tqVOnZtT3SU/HEQ6HM+p7BKnBhgAAAHDKAAAAsCEAAABiQwAAAMSGAAAAiA0BAAAQGwIAACA2BAAAQNL/B3Ok0swvdWm+AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O433EIsnZDeQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "7a25107a-7d39-4ed3-831d-f5f61305ce8f"
      },
      "source": [
        "wandb.init(project='loss-landscape', id='resnet20v1_aug_investigations')\n",
        "wandb.log({'prediction_disagreement': wandb.Image('prediction_disagreement.png')})\n",
        "wandb.log({'functional_similarity': wandb.Image('functional_similarity.png')})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/loss-landscape\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/loss-landscape/runs/resnet20v1_aug_investigations\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape/runs/resnet20v1_aug_investigations</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r6DMaLDkTZiJ",
        "colab_type": "text"
      },
      "source": [
        "## Similarity of Functions Across Randomly Initialized Trajectories"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oIm7JFxXTjkE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Retrieve different weights\n",
        "!mkdir ResNet20v1_CIFAR10_Aug_Diff_Inits\n",
        "!gsutil -m cp -r gs://losslandscape/ResNet20v1_CIFAR10_Aug_Diff_Inits/resnet20v1-aug-diff-inits-*.h5 ResNet20v1_CIFAR10_Aug_Diff_Inits/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mZhm6Qq4WGvA",
        "colab_type": "text"
      },
      "source": [
        "### Disagreement of predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ghi8mJWGUKDW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "62bf600d-719b-499d-ea01-77aa4f7a828c"
      },
      "source": [
        "independent_ckpts = os.listdir('ResNet20v1_CIFAR10_Aug_Diff_Inits/')\n",
        "independent_ckpts[:5]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['resnet20v1-aug-diff-inits-3.h5',\n",
              " 'resnet20v1-aug-diff-inits-2.h5',\n",
              " 'resnet20v1-aug-diff-inits-6.h5',\n",
              " 'resnet20v1-aug-diff-inits-4.h5',\n",
              " 'resnet20v1-aug-diff-inits-8.h5']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1IIKZkCUbi6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "86e2122deb8b43e88af03263672e90ae",
            "5796ccdf00e7459fa6b3e01f3c4eb82c",
            "c0f21fc88848419a9f6ab9c1e42cf823",
            "a462d96f40334d4aa3aa79a07523696d",
            "010a217c2a1049ab977243fc2533825b",
            "bcfd4bdf4db0427ba483c43a67d6df62",
            "fa913963a66049b389737d953763af92",
            "2751e8f7376345bd800ea45b6dbe74f6"
          ]
        },
        "outputId": "8e4820ee-6e8a-480f-d492-37cb4eaad807"
      },
      "source": [
        "predictions = []\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model('ResNet20v1_CIFAR10_Aug_Diff_Inits/' + independent_ckpts[i])\n",
        "  # get predictions for model\n",
        "  _, preds = evaluate(test_ds, model)\n",
        "\n",
        "  predictions.append(preds)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "86e2122deb8b43e88af03263672e90ae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVodr56SU-mY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "daaac65a8891426e804b67dddd1dc3ea",
            "fb45c39f03564158b13b5db1dd9dae46",
            "c44aa148bbce4b7c8496c6efabc4b73e",
            "98bc2de090b74af0bc793b280268cde9",
            "8c7ebaf2817c4932865fb5665a4f39a6",
            "c484a09b31a8442c8ed395bbdcf990e4",
            "c6dc5b46680d4ebc8db54c5d478a6cb1",
            "0707ebfe358d4a65acf4e9222f62769b"
          ]
        },
        "outputId": "47e995d6-dd60-47e3-db85-a02cca8907a2"
      },
      "source": [
        "empty_arr = np.zeros(shape=(10,10))\n",
        "\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  preds1 = predictions[i]\n",
        "  for j in range(i, 10):\n",
        "    preds2 = predictions[j]\n",
        "    \n",
        "    # compute dissimilarity\n",
        "    dissimilarity_score = 1-np.sum(np.equal(preds1, preds2))/10000 \n",
        "    \n",
        "    empty_arr[i][j] = dissimilarity_score\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = dissimilarity_score\n",
        "\n",
        "dissimilarity_coeff = empty_arr[::-1]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "daaac65a8891426e804b67dddd1dc3ea",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PVlC4Yg7VeeR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "2ab4c2c4-57b0-4035-87af-2161a1c69412"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns.heatmap(dissimilarity_coeff, cmap='RdBu_r');\n",
        "plt.xticks([0,2,4,6,8],[0,2,4,6,8]);\n",
        "plt.yticks([0,2,4,6,8],[8,6,4,2,1]);\n",
        "\n",
        "plt.savefig('independent_prediction_disagreement.png')"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAHYCAYAAAAyFvBfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfXBU5f338c/uQiQ+rLhowsZEqVpxq0ZHO2XoT7E/zNPUxQVbzBid0rENvVsebK0PaG1CrNrGe6ZKCWh1ig2DtU6mHSkZJiLFuSXWauuN+BCxFZMGJA+aGFciErK79x/eP3ptl2wSm91z0uv9mtkZcvZkr+8JhP3u57rOOZ5EIpEQAACwmtfpAgAAgPNoCAAAAA0BAACgIQAAAKIhAAAAoiEAAACiIQAAABpnQ3Do0CG9/vrrikajmaoHAAA4IG1D8Mwzz+iSSy5RRUWFXn31VX31q1/VbbfdptLSUu3YsSNbNQIAgAybku7JdevW6YknnlA0GlV1dbUeeughXXLJJdq7d69++MMfav78+eMa7O7cc/6tYjPpwCfDTpcwqoJpaf+6HOfzeJwuYVKbOc3ndAlpvXc45nQJaZ1zYo7TJYzK7T/Dmz73ZadLSGto14asjPO/PLMm/DUfTnRM+GtOtFHfYWbPni1JOuGEE3TJJZdIks4+++zMVgUAALIq7ZSBx+PR3r17tWvXLn388cd65ZVXJEnt7e2Kxdzd6QIAgLFLmxCsXLlS1113nbxerx544AGtWbNG7733nrq7u7V69eoslQgAQPb4LJ39TNsQ/Pd//7deeumlo19/6Utf0ptvvqmZM2fq1FNPzXhxAABkm63roca1Ss3n8+mCCy7IVC0AAMAh7l62DgBAltk6ZcCVCgEAAAkBAAAm1hAAAACmDAAAgL1ICAAAMNg6ZUBCAAAASAgAADCxhgAAAFiLhAAAAIOtawhoCAAAMNgandt63AAAwEBCAACAwdYpAxICAABAQgAAgMnW0w5pCAAAMDBlAAAArEVCAACAwdYpAxICAABAQgAAgMnWNQQ0BAAAGJgyAAAA1iIhAADAYOuUAQkBAAAgIQAAwGTrGgIaAgAADLY2BEwZAAAAEgIAAEwsKgQAANYiIQAAwMAaAgAAYC0SAgAADLauIaAhAADAwJQBAACwFgkBAAAGW6cMSAgAAAAJAQAAJlvXEGS1IQjk+LI53LjMnOb+3ui9wzGnS0jrxCnuDpxOnuru+nJt/V9oghwcjjtdwqhu+tyXnS4hrTXtf3K6BFdgygAAAFjL/R+LAQDIIi8JAQAAsBUJAQAABo+l63loCAAAMHgtbQiYMgAAACQEAACYPD47PyvbedQAACAJCQEAAAZbFxWSEAAAABoCAABMXp9nwh/j1d7ersrKSpWXl6uyslIdHR0p+7S2tuqaa67RBRdcoPr6+qTn1q1bp6uuukoLFizQNddco507d446JlMGAAAYPF7nPyvX1taqqqpKkUhEmzdvVk1NjTZu3Ji0T1FRke699161tLRoaGgo6bni4mLdeOONys3N1Z49e3TDDTeotbVV06ZNG3FM548aAAAc1dfXp7a2NoXDYUlSOBxWW1ub+vv7k/Y788wzFQqFNGVK6mf7yy+/XLm5uZKk2bNnK5FIaGBgIO24JAQAABgycWGiaDSqaDSast3v98vv9ydt6+rqUn5+vny+T+8Q7PP5lJeXp66uLgUCgXGP/dRTT+mMM87QzJkz0+5HQwAAQIY1NjaqoaEhZfvy5cu1YsWKjI370ksvac2aNdqwYcOo+9IQAABgyMRph0uWLNGiRYtStv9rOiBJwWBQPT09isVi8vl8isVi6u3tVTAYHNeYu3bt0q233qr169frrLPOGnV/GgIAAAyZuFLhsaYGRjJjxgyFQiE1NzcrEomoublZoVBoXNMFr776qn7wgx/oF7/4hc4///wxfQ+LCgEAcJnVq1dr06ZNKi8v16ZNm1RXVydJqq6u1muvvSZJ+utf/6p58+bpscce029/+1vNmzfv6OmFdXV1+uSTT1RTU6NIJKJIJKK33nor7ZieRCKRyOxh/VPDybOzNdS45Xjdf2Wq9w7HnC4hrROnuLu/PHmqu+vLdfnV0Q4Ox50uIS2fx90/P0laesZcp0tIa037n5wuIa3vDuzJyjjPzfnyhL/mvBfd/bOVSAgAAIBYQwAAQBLPJEiMM4GGAAAAg5fbHwMAAFuREAAAYOD2xwAAwFokBAAAGEgIAACAtUgIAAAw2HqWAQ0BAAAGpgwAAIC1SAgAADB4Lb1SIQkBAAAgIQAAwORhUSEAAPCyqBAAANiKhAAAAAOnHQIAAGuREAAAYGBRIQAAYFEhAACwFwkBAAAGD1cqHN3g4KDeeOMNHTx4MFP1AAAAB6RtCGpqatTf3y9Jevnll1VaWqrbbrtNpaWlam1tzUqBAABkk9fnnfDHZJB2yuCVV15RIBCQJK1Zs0YPP/ywiouL1d7erh/+8Ie67LLLxjXYh0fin73SDMudBItIjiQSTpeQ1sFh9/79StLJU939S9k/5O6fX/9QzOkS0qqbfbnTJYzqkc4XnC4hrROP8zldAhyUtiE4fPjw0T8PDg6quLhYkvS5z31OR44cyWxlAAA4gAsTHcPcuXP1s5/9TIcOHdKcOXO0detWSdLzzz+v6dOnZ6VAAACyyePzTvhjMkhb5Z133qnh4WHNmzdPzzzzjG6++WZdcMEF2rBhg+67775s1QgAADIs7ZRBTk6O7rrrLt18883q7OxUPB5XMBjUKaeckq36AADIKo93cnyin2hjug7B8ccfr/POOy/TtQAAAIdwYSIAAAyT5TTBiUZDAACAYbIsApxodh41AABIQkIAAICBhAAAAFiLhAAAAAOnHQIAAHl8dt7Twc42CAAAJCEhAADAwKJCAABgLRICAAAMXksXFdp51AAAIAkJAQAABlvXENAQAABgsLUhsPOoAQBAEhICAAAMtl6p0M6jBgAASUgIAAAw2LqGgIYAAACDrQ2BnUcNAACSkBAAAGDwkhAAAABbkRAAAGCw9bRDGgIAAAwsKgQAAK7Q3t6uyspKlZeXq7KyUh0dHSn7tLa26pprrtEFF1yg+vr6pOdisZjq6upUUlKi0tJSNTU1jTomCQEAAAY3JAS1tbWqqqpSJBLR5s2bVVNTo40bNybtU1RUpHvvvVctLS0aGhpKem7Lli3q7OzUtm3bNDAwoIULF2ru3LkqLCwccUznjxoAABzV19entrY2hcNhSVI4HFZbW5v6+/uT9jvzzDMVCoU0ZUrqZ/utW7dq8eLF8nq9CgQCKikpUUtLS9pxSQgAADBkYlFhNBpVNBpN2e73++X3+5O2dXV1KT8/Xz6fT5Lk8/mUl5enrq4uBQKBMY3X1dWlgoKCo18Hg0F1d3en/R4aAgAAMqyxsVENDQ0p25cvX64VK1Y4UFEqGgIAAAze///JfCItuX6JFi1alLL9X9MB6dNP8z09PYrFYvL5fIrFYurt7VUwGBzzeMFgUAcOHFBxcbGk1MTgWGgIAAAwZGJR4bGmBkYyY8YMhUIhNTc3KxKJqLm5WaFQaMzTBZJUUVGhpqYmlZWVaWBgQNu3b9fjjz+e9ntYVAgAgMusXr1amzZtUnl5uTZt2qS6ujpJUnV1tV577TVJ0l//+lfNmzdPjz32mH77299q3rx52rlzpyQpEomosLBQZWVluvbaa7Vs2TIVFRWlHdOTSCQSmT2sf7r3+M9na6hxy/V5nC5hVNHhuNMlpDXV4+6f4em57g7EDsWy9qv4mfQPxZwuIa262Zc7XcKoHul8wekS0jpxirs/I369542sjPPhr+6a8Nc8+Vv3TPhrTjR3/+0DAICscPdHJgAAsox7GQAAAFdcqdAJdh41AABIktWEYDDm5kVx7u+NYtlb//mZDMXdXZ/bf34HXb5o1O2L9mrf2ul0CaOKTZv489sn0tsHh0bfyQIkBAAAwFqsIQAAwMCiQgAAII/X3VM7mWJnGwQAAJKQEAAAYCIhAAAAtiIhAADAZOmiQjuPGgAAJCEhAADA4PHZuYaAhgAAABOLCgEAgK1ICAAAMJEQAAAAW5EQAABg4F4GAACAKQMAAGAvEgIAAEwkBAAAwFYkBAAAGFhUCAAAmDIAAAD2IiEAAMBEQgAAAGxFQgAAgMHW2x+TEAAAABICAACScNohAABgUSEAALAWCQEAAAYPCQEAALAVCQEAACYWFQIAAKYMAACAtUgIAAAwkRAAAABbkRAAAGBiUSEAAODmRgAAwFokBAAAmFhUCAAAbEVCAACAiYRgbP70pz9log4AAFzB4/VO+GMySJsQvP322ynb7rjjDm3YsEGJRELnnHNOxgoDAADZk7YhCIfDOv3005VIJI5ue//991VdXS2Px6M//vGP4xqsKHfqZ6syCz48EnO6hFHlH+fuGZ6Dw3GnS0hr36Fhp0tI677z5jldQlr3/r3V6RLSOjHH/THviVMmxydF61k6ZZD2HWb58uXavXu36urqVFBQIEmaP3++duzYkZXiAABAdozaELS1tenmm29WJBLRddddJ4/Hk63aAADIPo+dSc6oR/2FL3xBGzdu1LvvvqtvfvObOnLkSDbqAgAAWTSmSemcnBzdcssteuWVV/TSSy9luiYAAJxjaUIwrlVqF198sS6++OJM1QIAgOMSNAQAAMAN2tvbtWrVKg0MDGj69Omqr6/XrFmzkvaJxWK65557tHPnTnk8Hi1dulSLFy+WJPX19emOO+5QV1eXhoeHNWfOHN11112aMmXkt3072yAAAEbi8U78Y5xqa2tVVVWlp59+WlVVVaqpqUnZZ8uWLers7NS2bdv05JNPau3atdq/f78k6eGHH9bZZ5+tLVu26A9/+IPeeOMNbdu2Le2YNAQAALhIX1+f2traFA6HJX16TaC2tjb19/cn7bd161YtXrxYXq9XgUBAJSUlamlpkSR5PB4NDg4qHo9raGhIR44cUX5+ftpxmTIAAMCUgdPro9GootFoyna/3y+/35+0raurS/n5+fL5Pr1Aks/nU15enrq6uhQIBJL2+59rBElSMBhUd3e3JOl73/ueVqxYocsuu0yHDh3S9ddfr0svvTRtjTQEAACYMnDvgcbGRjU0NKRsX758uVasWDHh47W0tGj27NlqbGzU4OCgqqur1dLSooqKihG/h4YAAIAMW7JkiRYtWpSy/V/TAenTT/o9PT2KxWLy+XyKxWLq7e1VMBhM2e/AgQMqLi6WlJwYbNq0Sffdd5+8Xq9OOukkzZ8/Xy+++GLahoA1BAAAGBIe74Q//H6/CgsLUx7HaghmzJihUCik5uZmSVJzc7NCoVDSdIEkVVRUqKmpSfF4XP39/dq+fbvKy8slSYWFhXruueckSUNDQ3rhhRf0+c9/Pu1x0xAAAOAyq1ev1qZNm1ReXq5Nmzaprq5OklRdXa3XXntNkhSJRFRYWKiysjJde+21WrZsmYqKiiRJd955p15++WUtWLBACxcu1KxZs3TttdemHdOTMG9lmGEPTT8vW0ON22S426Hb75Tm9rsdfhLP2j/1z4S7Hf573P77IUmBHHfX6Pbf4W/178nKOMPvvjnhrznl9NCEv+ZEYw0BAAAmS69UaOdRAwCAJCQEAACYSAgAAICtSAgAADDYerdDO48aAAAkISEAAMBkaUJAQwAAgCkDNzeaDOxsgwAAQBISAgAATJZOGdh51AAAIAkJAQAABltPO6QhAADA5LWzIbDzqAEAQBISAgAATJZOGdh51AAAIAkJAQAAJksTAhoCAABMljYEdh41AABIQkIAAIDB1usQ2HnUAAAgCQkBAAAmEgIAAGArEgIAAEwej9MVOIKGAAAAE1MGAADAViQEAAAYOO0QAABYK6sJwd7BoWwONy6nTPU5XcKo+odiTpeQVs/hYadLSOvRi+Y7XUJad+55zukS0grkujtQ9E2ChWCHYgmnS0ir+xN3/x+TNZYmBO7+DQcAIMsSk6C5zAQ72yAAAJCEhAAAAEPC3TM7GUNCAAAASAgAADDFLY0IaAgAADDY2Q4wZQAAAERCAABAkrilEQEJAQAAICEAAMCUsHRRIQkBAAAgIQAAwGTrGgIaAgAADJb2A0wZAAAAEgIAAJLYOmVAQgAAAEgIAAAw2XraIQ0BAACGuNMFOIQpAwAAQEIAAIDJ0hkDEgIAAEBCAABAEltPO6QhAADAYOtZBkwZAAAAEgIAAEycdggAAFyhvb1dlZWVKi8vV2VlpTo6OlL2icViqqurU0lJiUpLS9XU1JT0/NatW7VgwQKFw2EtWLBA77//ftoxSQgAADC4YQlBbW2tqqqqFIlEtHnzZtXU1Gjjxo1J+2zZskWdnZ3atm2bBgYGtHDhQs2dO1eFhYV67bXX1NDQoMbGRp122mn66KOPlJOTk3ZMEgIAADIsGo1q//79KY9oNJqyb19fn9ra2hQOhyVJ4XBYbW1t6u/vT9pv69atWrx4sbxerwKBgEpKStTS0iJJ+vWvf60bb7xRp512miTppJNO0nHHHZe2RhICAAAM8QxEBI2NjWpoaEjZvnz5cq1YsSJpW1dXl/Lz8+Xz+SRJPp9PeXl56urqUiAQSNqvoKDg6NfBYFDd3d2SpL1796qwsFDXX3+9Pv74Y5WWluq73/2uPB7PiDXSEAAAYMjEjMGSJUu0aNGilO1+vz8Do326vuCtt97SY489pqGhIX37299WQUGBFi5cOOL30BAAAJBhfr9/zG/+wWBQPT09isVi8vl8isVi6u3tVTAYTNnvwIEDKi4ulpScGBQUFKiiokI5OTnKycnRlVdeqVdffTVtQ8AaAgAADPHExD/GY8aMGQqFQmpubpYkNTc3KxQKJU0XSFJFRYWampoUj8fV39+v7du3q7y8XNKn6w5aW1uVSCR05MgR/fnPf9Z5552XdlwSAgAAXGb16tVatWqV1q9fL7/fr/r6eklSdXW1Vq5cqQsvvFCRSES7d+9WWVmZJGnZsmUqKiqSJF111VV6/fXX9dWvflVer1eXXXaZvv71r6cd05PI4jUab5l6VraGGrdTpvqcLmFUMTecC5NGz+Fhp0tI69GL5jtdQlp37nnO6RLSKsp19+cHX5rFUhibdw+5+3f4Rx//PSvj/L33owl/zc/nnTThrznR3P0bDgBAlsUzsqzQ/VhDAAAASAgAADC5fHY2Y0gIAAAACQEAAKbxnib4n4KGAAAAA1MGAADAWiQEAAAYOO3wGD744AP96Ec/0o033qjHH3886bl/vTsTAACYvNImBLW1tSosLNQVV1yhJ554Qi+88IIefPBBTZkyRfv27Rv3YKfmuPdqgB0fH3G6hFHNnObuQMftVwKs3r3D6RLSmpY71ekS0to76O7fkRN87p8BPXGK+2sEawiOqaOjQ7fddpvKysq0YcMGnXbaafrOd76jw4cPZ6s+AACQBWkbgiNH/vmJwOPxqLa2Vueee66WLl1KUwAA+I8UTyQm/DEZpG0IioqK9Je//CVp2+23366LLrpIHR0dmawLAABHxOIT/5gM0k5K33///fIc4w5iN998s66++uqMFQUAALIrbUMwffr0EZ8755xzJrwYAACcNlki/onGklcAAMCFiQAAMMUsTQhoCAAAMDBlAAAArEVCAACAYbKcJjjRSAgAAAAJAQAAJlvXENAQAABgsPUsA6YMAAAACQEAAKa4nQEBCQEAACAhAAAgSczSiICEAAAAkBAAAGDitEMAAKCYnf0AUwYAAICEAACAJLZOGZAQAAAAEgIAAEy2nnZIQwAAgIEpAwAAYC0SAgAADJx2CAAArEVCAACAwdY1BDQEAAAY4paeZcCUAQAAICEAAMDEokIAAGAtEgIAAAy2LiokIQAAACQEAACYYpYmBDQEAAAYOO0QAABYi4QAAAADpx0CAABrkRAAAGCw9bRDGgIAAAycZZAFQy5euRk66TinSxjVrWf/l9MlpHXnnuecLiEt73Hu7n9PnOLuGbzBWNzpEtI6Mgn+Ez847O6foa1vhPiUu/+HBAAgy2Iu/vCaSe7+SAIAgIXa29tVWVmp8vJyVVZWqqOjI2WfWCymuro6lZSUqLS0VE1NTSn7vPPOO7roootUX18/6pg0BAAAGGLxxIQ/xqu2tlZVVVV6+umnVVVVpZqampR9tmzZos7OTm3btk1PPvmk1q5dq/379//zOGIx1dbWqqSkZExj0hAAAGDIREMQjUa1f//+lEc0Gk0Zv6+vT21tbQqHw5KkcDistrY29ff3J+23detWLV68WF6vV4FAQCUlJWppaTn6/COPPKKvfOUrmjVr1piOm4YAAIAMa2xs1JVXXpnyaGxsTNm3q6tL+fn58vl8kiSfz6e8vDx1dXWl7FdQUHD062AwqO7ubknSnj171Nraqm9+85tjrpFFhQAAGDKxqHDJkiVatGhRyna/3z/hYx05ckQ//vGP9dOf/vRoUzEWNAQAAGSY3+8f85t/MBhUT0+PYrGYfD6fYrGYent7FQwGU/Y7cOCAiouLJf0zMXjvvffU2dmppUuXSpKi0agSiYQOHjyon/zkJyOOS0MAAIDB6dMOZ8yYoVAopObmZkUiETU3NysUCikQCCTtV1FRoaamJpWVlWlgYEDbt2/X448/roKCAr344otH91u7dq0+/vhj3X777WnHpSEAAMDgdEMgSatXr9aqVau0fv16+f3+o6cNVldXa+XKlbrwwgsViUS0e/dulZWVSZKWLVumoqKizzymJ5HI3qWp7s49J1tDjdvJU8c+z+IUrlT473H7CtpAjrv/DfYcHna6hLR8Ho/TJYxqqstrdPuVCmsOvZ2Vce5+5q0Jf82a0tkT/poTjYQAAACDGxICJ7j9QxMAAMgCEgIAAAwkBAAAwFokBAAAGGxNCGgIAAAw2NoQMGUAAABICAAAMA2TEAAAAFuREAAAYLB1DQENAQAABlsbAqYMAAAACQEAACa33+QpU0gIAAAACQEAACZb1xDQEAAAYLC1IWDKAAAAkBAAAGAiIQAAANYiIQAAwBCLx50uwREkBAAAgIQAAACTrWsIaAgAADDY2hAwZQAAAEgIAAAwDZMQAAAAW5EQAABgsHUNAQ0BAAAGWxsCpgwAAAAJAQAAJhICAABgLRICAAAMtiYENAQAABhoCMZpwYIF2rJly7i+50jCvT/kW8/+L6dLGNX/3vu80yWk9eYnw06XkFZR7lSnS0jr4LC777A2zevuGcZcn8fpEkZ14hR3/wz/78AnTpcAB6VtCN5+++0Rn/vggw8mvBgAAJyWICFIFQ6HdfrppytxjE/2AwMDGSsKAABkV9qG4PTTT9dvfvMb5efnpzx3xRVXZKwoAACcErc0IUg7oVVWVqZ33333mM+VlpZmpCAAAJB9aROC22+/fcTn7rrrrgkvBgAApx1rmtwGnHYIAIDB1kWF7j4HBgAAZAUJAQAABhYVAgAAa5EQAABgSLj7oqEZQ0MAAIDB1rMMmDIAAAAkBAAAmFhUCAAArEVCAACAwdYLE9EQAABgsLUhYMoAAACQEAAAYIpz2iEAALAVCQEAAAbWEAAAAFdob29XZWWlysvLVVlZqY6OjpR9YrGY6urqVFJSotLSUjU1NR19bt26dbrqqqu0YMECXXPNNdq5c+eoY5IQAABgcENCUFtbq6qqKkUiEW3evFk1NTXauHFj0j5btmxRZ2entm3bpoGBAS1cuFBz585VYWGhiouLdeONNyo3N1d79uzRDTfcoNbWVk2bNm3EMUkIAAAwxOOJCX9Eo1Ht378/5RGNRlPG7+vrU1tbm8LhsCQpHA6rra1N/f39Sftt3bpVixcvltfrVSAQUElJiVpaWiRJl19+uXJzcyVJs2fPViKR0MDAQNrjJiEAACDDGhsb1dDQkLJ9+fLlWrFiRdK2rq4u5efny+fzSZJ8Pp/y8vLU1dWlQCCQtF9BQcHRr4PBoLq7u1PGeOqpp3TGGWdo5syZaWukIQAAwJCJux0uWbJEixYtStnu9/snfCzTSy+9pDVr1mjDhg2j7ktDAABAhvn9/jG/+QeDQfX09CgWi8nn8ykWi6m3t1fBYDBlvwMHDqi4uFhSamKwa9cu3XrrrVq/fr3OOuusUcdlDQEAAIZEfOIf4zFjxgyFQiE1NzdLkpqbmxUKhZKmCySpoqJCTU1Nisfj6u/v1/bt21VeXi5JevXVV/WDH/xAv/jFL3T++eePaVwSAgAADG64/fHq1au1atUqrV+/Xn6/X/X19ZKk6upqrVy5UhdeeKEikYh2796tsrIySdKyZctUVFQkSaqrq9Mnn3yimpqao695//33a/bs2SOOSUMAAIDLnH322UnXFfgfjz766NE/+3w+1dXVHfP7f/e73417TBoCAAAMbrgOgRNYQwAAAEgIAAAw2ZoQ0BAAAGDg9scAAMBaJAQAABhsnTIgIQAAACQEAACYSAgAAIC1SAgAADC44dLFTqAhAADAkInbH08GTBkAAAASAgAATLYuKsxqQ1AfuiKbw43L7W/+H6dLGNWHHo/TJaQ16/ipTpeQ1vtDMadLSOvsE3KcLiGtfYeOOF3CKNwfeOZ43f1GUzCNz4g2428fAAADiwoBAIAScXeniZni/owNAABkHAkBAAAGEgIAAGAtEgIAAAy2JgQ0BAAAGBIxOxsCpgwAAAAJAQAAJlunDEgIAAAACQEAACYSAgAAYC0SAgAADLYmBDQEAAAYbG0ImDIAAAAkBAAAmEgIAACAtUgIAAAwxC1NCGgIAAAwMGUAAACsRUIAAICBhAAAAFiLhAAAAEMiZmdCQEMAAICBKQMAAGAtEgIAAAwkBAAAwFokBAAAGEgIAACAtUgIAAAwJOJxp0twBA0BAAAGpgwAAIC1SAgAADCQEAAAAGuREAAAYIhbmhDQEAAAYLD15kZMGQAAABICAABMLCoEAADWIiEAAMBAQgAAAJSIxyb8MV7t7e2qrKxUeXm5Kisr1dHRkbJPLBZTXV2dSkpKVFpaqqampjE9NxIaAgAAXKa2tlZVVVV6+umnVVVVpZqampR9tmzZos7OTm3btk1PPvmk1q5dq/3794/63EhoCAAAMGQiIYhGo9q/fy+KYXAAAATRSURBVH/KIxqNpozf19entrY2hcNhSVI4HFZbW5v6+/uT9tu6dasWL14sr9erQCCgkpIStbS0jPrcSLK6hmBo14ZsDgcAwLhl4r1q7dq1amhoSNm+fPlyrVixImlbV1eX8vPz5fP5JEk+n095eXnq6upSIBBI2q+goODo18FgUN3d3aM+NxIWFQIAkGFLlizRokWLUrb7/X4Hqjk2GgIAADLM7/eP+c0/GAyqp6dHsVhMPp9PsVhMvb29CgaDKfsdOHBAxcXFkpJTgXTPjYQ1BAAAuMiMGTMUCoXU3NwsSWpublYoFEqaLpCkiooKNTU1KR6Pq7+/X9u3b1d5efmoz43Ek0gkEpk5JAAA8Fns3btXq1atUjQald/vV319vc466yxVV1dr5cqVuvDCCxWLxXT33Xfr+eeflyRVV1ersrJSktI+NxIaAgAAwJQBAACgIQAAAKIhAAAAoiEAAACiIQAAAMpSQzCWuzbh2D744ANVV1ervLxcCxYs0PLly1OuZ42xaWho0OzZs/W3v/3N6VImncOHD6u2tlZlZWVasGCBfvzjHztd0qTy7LPPauHChYpEIrr66qu1bds2p0sCUmTltMNvfOMb+trXvqZIJKLNmzfrd7/7nTZu3JjpYf8jDAwM6K233tKcOXMkSfX19frwww913333OVzZ5PLGG2/ogQce0DvvvKOHH35Y5557rtMlTSr33HOPvF6v7rjjDnk8Hr3//vs69dRTnS5rUkgkEvrSl76kxx9/XOeee6727Nmj6667Ti+//LK8XkJauEfG/zWO9a5NOLbp06cfbQYk6eKLL9aBAwccrGjyGRoa0t13363Vq1c7XcqkNDg4qKeeeko33XSTPB6PJNEMjJPX69VHH30kSfroo4+Ul5dHMwDXyfi9DMZ61yaMLh6P64knntD8+fOdLmVSWbNmja6++moVFhY6XcqktG/fPk2fPl0NDQ168cUXdcIJJ+imm27SF7/4RadLmxQ8Ho8efPBBfe9739Pxxx+vwcFBPfLII06XBaSgRZ1EfvKTn+j444/XDTfc4HQpk8auXbv0+uuvq6qqyulSJq1YLKZ9+/bpC1/4gn7/+9/rlltu0YoVK3Tw4EGnS5sUhoeH9ctf/lLr16/Xs88+q4ceekjf//73NTg46HRpQJKMNwTmXZskjXjXJqRXX1+vf/zjH3rwwQeJGsfhL3/5i/bu3asrr7xS8+fPV3d3t771rW+ptbXV6dImjWAwqClTphyd9rvooot0yimnqL293eHKJoc333xTvb29uvTSSyVJl156qXJzc7V3716HKwOSZfydZax3bcLIfv7zn+v111/XunXrlJOT43Q5k8rSpUvV2tqqHTt2aMeOHZo5c6Z+9atf6bLLLnO6tEkjEAhozpw5R2+S0t7err6+Pp155pkOVzY5zJw5U93d3XrnnXckfXrTmr6+Pp1xxhkOVwYky8pZBiPdtQmj+/vf/65wOKxZs2Zp2rRpkqTCwkKtW7fO4comp/nz53OWwWewb98+3XnnnRoYGNCUKVP0/e9/X1dccYXTZU0af/jDH/Too48eXZS5cuVKlZSUOFwVkIy7HQIAABYVAgAAGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAg6f8BqTcKr6bY6WkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "27BlZOgxWNEo",
        "colab_type": "text"
      },
      "source": [
        "### Cosine Similarity of weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28Ud1Y9SWJkf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "305439c4cb1547eebce197eaf29fad2a",
            "2c9b073f9b62437aafebcbae58d41821",
            "14eb209382ee423ba536ac9fe8391b07",
            "c0c97d1c43d4471ab048dd8d4a5be37e",
            "88defc245af94f3ebd2ee3aaf56b8e3b",
            "a9dd9992bd3e436f9552698e7e854995",
            "0dcbc1e5522b45039a54ce5fc869c8dc",
            "d3d1fd014af640828c96324f4cb88e61"
          ]
        },
        "outputId": "3ed5b831-2468-4df3-fec9-12804c799877"
      },
      "source": [
        "weights_of_models = []\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model('ResNet20v1_CIFAR10_Aug_Diff_Inits/' + independent_ckpts[i])\n",
        "  # get predictions for model\n",
        "  weights = get_model_weights(model)\n",
        "\n",
        "  weights_of_models.append(weights)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "305439c4cb1547eebce197eaf29fad2a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5sTlAuopWemT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "275cf834720e4393bd0ef4d6d3458126",
            "7c9faafacfac4c5ab1a1fbdce27cb9a8",
            "8bef4c136b814da6ad3743a490900112",
            "b39d02e7d8d14f278bcdd8d34fc883c8",
            "9c384e499e2d4a7c80f482c2dd0c882f",
            "952d665b6f5a440885eda642dafc52da",
            "475a3b31c90e4bfd860e6bb693f43e72",
            "b53854aee89646fd8fc0b387b908d879"
          ]
        },
        "outputId": "d9fdf96b-e136-4d97-e7b7-e1bbbb53c027"
      },
      "source": [
        "empty_arr = np.zeros(shape=(10,10))\n",
        "\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  weights1 = weights_of_models[i]\n",
        "  for j in range(i, 10):\n",
        "    weights2 = weights_of_models[j]\n",
        "    \n",
        "    # compute cosine similarity of weights\n",
        "    cos_sim = np.dot(weights1, weights2)/(norm(weights1)*norm(weights2))\n",
        "    \n",
        "    empty_arr[i][j] = cos_sim\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = cos_sim\n",
        "\n",
        "cos_sim_coeff = empty_arr[::-1]"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "275cf834720e4393bd0ef4d6d3458126",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CqDNUbaYWisq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "33b0bf8e-7fa3-4602-f44c-d9c081946265"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns.heatmap(cos_sim_coeff, cmap='RdBu_r');\n",
        "plt.xticks([0,2,4,6,8],[0,2,4,6,8]);\n",
        "plt.yticks([0,2,4,6,8],[8,6,4,2,1]);\n",
        "\n",
        "plt.savefig('independent_functional_similarity.png')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAHYCAYAAABHrTENAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df3BU1f3/8dfuYipatyGRhA1oUSiwKkjRgXGK0oYfYexiSK2lRsdU2lh/JKhoMVLNj2KpcTooNSCVKRgm1HGorchKBRQ7fkItWvwBEkHFUECXRBLSlR8KbPb7h518z7pkQzCb3b33+ZjZmeTk5ux7SfSd17nn3nWEw+GwAACA5TkTXQAAAOgdNH0AAGyCpg8AgE3Q9AEAsAmaPgAANkHTBwDAJmj6AAAkkerqauXm5mr48OF6//33T3pMKBRSVVWVJk2apMmTJ2vVqlWnNHe3mv7Ro0f17rvvKhgMdufbAADAKZo4caJWrlypgQMHdnrMmjVrtGfPHq1fv17PPPOMHn/8ce3bt6/LuWM2/Q0bNmjMmDGaOnWqtm7dqquvvlpz5szR5MmTtXHjxu6/EgAAbCgYDGrfvn1Rj5OF6Msvv1wejyfmfGvXrtV1110np9OpjIwMTZo0SS+++GKXdfSJ9cVFixbp6aefVjAYVHFxsZ544gmNGTNGu3bt0j333KPc3Nwun8D0jcuKu3V8bwq3hxJdQpccTleiS0AcJfvvIL9/1nfzmxsSXUJMS8K7e+V5bnUM7vE5L/7DPaqpqYkaLykpUWlpabfnCwQCysnJ6fjc4/Fo//79XX5fzKYvScOHD5cknX322RozZowkaciQId0uEAAAuyoqKlJBQUHUuNvt7tU6YjZ9h8OhXbt2KRgM6siRI3r77bc1evRoNTY2KhRK7lQCAECycLvdPdrgPR6PPvnkE40aNUpSdPLvTMymP2vWLF1//fVyOp169NFHtXDhQn366afav3+/Kisre6RwAACSicuR6Aq6NnXqVK1atUpTpkxRW1ubXnrpJa1cubLL74vZ9H/wgx/o9ddf7/h87Nixeu+99zRgwACde+65X79qAACSjMuR2K7/0EMPaf369Tpw4IBuvvlmpaen64UXXlBxcbFmzZqlkSNHKj8/X++8846mTJkiSbrjjjt03nnndTm3ozffWpeNfF8PG6msLdl/B/n9sz428n1pluuCHp/zD6HGHp/zdHS5kQ8AADtJheX908Ud+QAAsAmSPgAAhkSf048nmj4AAAaW9wEAQMoj6QMAYLDy8j5JHwAAmyDpAwBg4Jw+AABIeSR9AAAMVj6nT9MHAMBg5SVwK782AABgIOkDAGCw8vI+SR8AAJsg6QMAYLDyJXs0fQAADCzvAwCAlEfSBwDAYOXlfZI+AAA2QdIHAMBg5XP6NH0AAAws7wMAgJRH0gcAwGDl5X2SPgAANkHSBwDAYOVz+jR9AAAMVm76LO8DAGATJH0AAAxs5AMAACmPpA8AgIFz+gAAIOWR9AEAMFj5nD5NHwAAA8v7AAAg5ZH0AQAwWHl5n6QPAIBNkPQBADBY+Zw+Tf9/HE5XokuAzfE7aH03v7kh0SXEtHzM5ESXENOSXnoelvcBAEDKI+kDAGBwkvQBAECqI+kDAGBwWHgnH00fAACD08JNn+V9AABsgqQPAIDB4Up8Hm5sbFRZWZna2tqUnp6u6upqDR48OOKYTz/9VOXl5dq3b59OnDihW2+9Vfn5+THnTfwrAwAAESoqKlRYWKh169apsLBQ5eXlUcc8/PDDuuSSS7RmzRqtXLlSjz76qAKBQMx5afoAABgcLkePP4LBoPbt2xf1CAaDUc/f0tKihoYG+Xw+SZLP51NDQ4NaW1sjjtuxY4euvPJKSVJGRoZGjBihv//97zFfG8v7AADEWW1trWpqaqLGS0pKVFpaGjEWCASUnZ0tl+vLu3S6XC5lZWUpEAgoIyOj47iLL75Ya9eu1ciRI7Vv3z699dZbGjRoUMw6aPoAABjisXu/qKhIBQUFUeNut/u05ywrK9P8+fOVn5+vnJwcXXHFFR1/KHSGpg8AgMHh7Pkz3263+5QbvMfjUVNTk0KhkFwul0KhkJqbm+XxeCKOy8jI0O9///uOz4uLizV06NCYc3NOHwCAJJKZmSmv1yu/3y9J8vv98nq9EUv7knTw4EGdOHFCkvTaa6/p/fff79gH0BmSPgAAhmS4OU9lZaXKysq0ePFiud1uVVdXS/oyzc+aNUsjR47U1q1b9dvf/lZOp1P9+vXTkiVL1Ldv35jz0vQBAEgyQ4YM0apVq6LGly5d2vHxhAkTNGHChG7NS9MHAMDAvfcBALCJZLgjX7xY95UBAIAIJH0AAAzJsJEvXkj6AADYBEkfAACDw2ndpE/TBwDA4GQjHwAASHUkfQAADFa+Tp+kDwCATZD0AQAwkPQBAEDKI+kDAGCw8u59mj4AAAaW9wEAQMoj6QMAYHBa+I58JH0AAGyCpA8AgMHBRj4AAOyBt9YFAAApj6QPAICBS/YAAEDKI+kDAGBgIx8AADbBRj4AAJDySPoAABgc3JHvS4cPH9b27dt16NCheNUDAADiJGbTLy8vV2trqyRpy5Ytmjx5subMmaPJkyervr6+VwoEAKA3OV3OHn8ki5jL+2+//bYyMjIkSQsXLtSSJUs0atQoNTY26p577tH48eN7pUh8KdweSnQJMTmcrkSXkNL4+X49N7+5IdEldGn5mMmJLgE2F7Ppf/HFFx0fHz58WKNGjZIkXXDBBTp+/Hh8KwMAIAFse3OeK664Qg8//LCOHj2qcePGae3atZKkTZs2KT09vVcKBACgNzlczh5/JIuYlcydO1cnTpzQVVddpQ0bNmj27Nm65JJLtGzZMs2fP7+3agQAAD0g5vJ+WlqaHnjgAc2ePVt79uxRe3u7PB6P+vXr11v1AQDQqxzO5EnmPe2UrtM/66yzNGLEiHjXAgAA4oib8wAAYEimS+x6Gk0fAABDMm2862nWfWUAACACSR8AAANJHwAApDySPgAABttfsgcAgF04XMn9PhNfh3X/nAEAABFI+gAAGNjIBwAAUh5JHwAAgzMJNvI1NjaqrKxMbW1tSk9PV3V1tQYPHhxxTEtLi+6//34FAgGdOHFC48aN0wMPPKA+fTpv7Yl/ZQAAIEJFRYUKCwu1bt06FRYWqry8POqYJUuWaMiQIVqzZo2ef/55bd++XevXr485L00fAACDw+Xs8Ud3tLS0qKGhQT6fT5Lk8/nU0NCg1tbWyDodDh0+fFjt7e06duyYjh8/ruzs7Jhzs7wPAIAhHhv5gsGggsFg1Ljb7Zbb7Y4YCwQCys7Olut/lw66XC5lZWUpEAgoIyOj47jbb79dpaWlGj9+vI4ePaobbrhBl112Wcw6SPoAAMRZbW2tJk6cGPWora097TlffPFFDR8+XPX19Xr11Vf173//Wy+++GLM7yHpAwBgiMcd+YqKilRQUBA1/tWUL0kej0dNTU0KhUJyuVwKhUJqbm6Wx+OJOK6urk7z58+X0+nUOeeco9zcXG3evFlTp07ttA6SPgAAceZ2uzVo0KCox8mafmZmprxer/x+vyTJ7/fL6/VGLO1L0qBBg/Tqq69Kko4dO6bXXntN3/nOd2LWQdMHAMCQ6I18klRZWam6ujrl5eWprq5OVVVVkqTi4mJt27ZNkjR37lxt2bJF06ZN0/Tp0zV48GD95Cc/iTkvy/sAABiS4Y58Q4YM0apVq6LGly5d2vHx+eefr+XLl3dr3sS/MgAA0CtI+gAAGJxJkPTjxbqvDAAARCDpAwBgiMcle8mCpg8AgCEZNvLFi3VfGQAAiEDSBwDAQNIHAAApj6QPAIDByhv5rPvKAABABJI+AAAG5//ex96KaPoAABjYyAcAAFIeSR8AAANJHwAApDySPgAABitfskfTBwDAwPI+AABIeb2a9MPtod58um5xOK17XWZvSeafr5T8P2Nnn7RElxDTz/79YqJLiGn5mMmJLgEWQdIHAAApj3P6AAAY2MgHAIBNJPupwK/Dun/OAACACCR9AABMJH0AAJDqSPoAAJgsvJHPuq8MAABEIOkDAGBwuKx7Tp+mDwCAiY18AAAg1ZH0AQAwkfQBAECqI+kDAGDg3vsAANgFy/sAACDVkfQBADCR9AEAQKoj6QMAYGAjHwAAdsHyPgAASHUkfQAATCR9AACQ6kj6AAAYrPzWuiR9AABsgqQPAIApCS7Za2xsVFlZmdra2pSenq7q6moNHjw44pg5c+Zo586dHZ/v3LlTixYt0sSJEzudl6YPAIApCTbyVVRUqLCwUPn5+Vq9erXKy8u1YsWKiGMeeeSRjo937NihoqIiXXnllTHnTfyfMwAAoENLS4saGhrk8/kkST6fTw0NDWptbe30e/7yl79o2rRpSktLizk3SR8AAIMjDkk/GAwqGAxGjbvdbrnd7oixQCCg7Oxsuf63odDlcikrK0uBQEAZGRlRcxw7dkxr1qzRU0891WUdNH0AAOKstrZWNTU1UeMlJSUqLS39WnO/9NJLysnJkdfr7fJYmj4AAKY4bOQrKipSQUFB1PhXU74keTweNTU1KRQKyeVyKRQKqbm5WR6P56RzP/vss7r22mtPqQ6aPgAAhngs759sGb8zmZmZ8nq98vv9ys/Pl9/vl9frPenS/v79+7VlyxYtWLDglOZmIx8AAEmmsrJSdXV1ysvLU11dnaqqqiRJxcXF2rZtW8dxf/vb3/SDH/xA3/rWt05pXpI+AACmJLhkb8iQIVq1alXU+NKlSyM+v+2227o1L0kfAACbIOkDAGBKgjvyxQtNHwAAA2+4AwAAUh5JHwAAUxJs5IsXkj4AADZB0gcAwETS///++c9/xqMOAACSgsPp7PFHsoiZ9D/88MOosfvvv1/Lli1TOBzW0KFD41YYAADoWTGbvs/n08CBAxUOhzvGDhw4oOLiYjkcDr388stxLxCpIx73q+5J4fZQokuI6eY3NyS6hJieunxqokuIKXziWKJL6BL/jaSIJP85fR0xm35JSYneeecdVVVVKScnR5KUm5urjRs39kpxAACg53TZ9BsaGjR79mzl5+fr+uuvl8Ph6K3aAADofY7kOQff07p8ZRdddJFWrFihjz/+WD/72c90/Pjx3qgLAAD0sFO6ZC8tLU333nuv3n77bb3++uvxrgkAgMSxcNLv1nX6o0eP1ujRo+NVCwAACRe2cNO37isDAAARuCMfAAAmkj4AAEh1JH0AAEwWvjSdpg8AgCmJ7pXf06z7ygAAQASSPgAABi7ZAwAAKY+kDwCAycJJn6YPAIDJwk3fuq8MAABEIOkDAGAi6QMAgFRH0gcAwMAlewAAIOWR9AEAMFk46dP0AQAwWfgNd6z75wwAAIhA0gcAwGTh5X3rvjIAABCBpA8AgMHKl+zR9AEAMDmt2/St+8oAAEAEkj4AACYLL+9b95UBAIAIJH0AAEwWTvo0fQAATBZu+tZ9ZQAAIAJJHwAAg5Wv07fuKwMAIEU1NjZqxowZysvL04wZM7R79+6THrd27VpNmzZNPp9P06ZN04EDB2LOS9IHAMCUBEm/oqJChYWFys/P1+rVq1VeXq4VK1ZEHLNt2zbV1NSotrZW/fv312effaa0tLSY8yb+lQEAgA4tLS1qaGiQz+eTJPl8PjU0NKi1tTXiuKeeekozZ85U//79JUnnnHOOvvGNb8Scm6QPAIDJ4ejxKYPBoILBYNS42+2W2+2OGAsEAsrOzpbL5ZIkuVwuZWVlKRAIKCMjo+O4Xbt2adCgQbrhhht05MgRTZ48WbfddpscMeqn6QMAYIrD8n5t7VOqqamJGi8pKVFpaelpzRkKhbRz504tX75cx44d0y9+8Qvl5ORo+vTpnX4PTR8AgDgrKipSQUFB1PhXU74keTweNTU1KRQKyeVyKRQKqbm5WR6PJ+K4nJwcTZ06VWlpaUpLS9PEiRO1devWmE2fc/oAABjCDmePP9xutwYNGhT1OFnTz8zMlNfrld/vlyT5/X55vd6IpX3py3P99fX1CofDOn78uP71r39pxIgRMV8bTR8AgCRTWVmpuro65eXlqa6uTlVVVZKk4uJibdu2TZL0wx/+UJmZmbr66qs1ffp0DR06VD/+8Y9jzusIh8PhuFf/P2nfndlbT4UEcDhdiS4hppvf3JDoEmJaNjo30SXElOw/X1jfF1uW9srzfH70aI/PeWbfvj0+5+ngnD4AAIZwHHbvJwuW9wEAsAmSPgAAht476d37SPoAANgESR8AAEO7haM+TR8AAIN1Wz7L+wAA2AZJHwAAQ7uFoz5JHwAAmyDpAwBg6MUb1fY6kj4AADZB0gcAwGDlc/o0fQAADBbu+SzvAwBgFyR9AAAMVl7eJ+kDAGATJH0AAAxWvmSPpg8AgKE90QXEEcv7AADYBEkfAACDhVf3SfoAANgFSR8AAIOVL9mj6QMAYLDy7n2W9wEAsAmSPgAABi7ZAwAAKY+kDwCAwcKn9En6AADYBUkfAABDu4WjPk0fAACDdVs+y/sAANgGSR8AAIOV78hH0gcAwCZI+gAAGCy8j4+mDwCAqd3CW/lY3gcAwCZI+gAAGKy8vE/SBwDAJkj6AAAYrHzJHk0fAAADy/sAACDlkfQBADDY9pK9gwcP6te//rVmzpyplStXRnyttLQ0roUBAICeFTPpV1RUaNCgQZowYYKefvppvfbaa3rsscfUp08f7d27t7dqRIq4+c0NiS4hpuVjJie6hJgciS6gC+H2UKJLiMnhdCW6BFiEbc/p7969W3PmzNGUKVO0bNky9e/fX7/85S/1xRdf9FZ9AACgh8Rs+sePH+/42OFwqKKiQsOGDdMtt9xC4wcAWFJ7ONzjj+5qbGzUjBkzlJeXpxkzZmj37t1Rxzz++OO64oorlJ+fr/z8fFVVVXU5b8ymf9555+mNN96IGLvvvvt06aWXnrQAAABSXai95x/dVVFRocLCQq1bt06FhYUqLy8/6XHTp0/X6tWrtXr1alVUVHQ5b8xz+o888ogcjugzjbNnz9Y111xziqUDAGBvwWBQwWAwatztdsvtdkeMtbS0qKGhQcuXL5ck+Xw+zZs3T62trcrIyPhadcRs+unp6Z1+bejQoV/riQEASEansxzfldraWtXU1ESNl5SURF0NFwgElJ2dLZfry82pLpdLWVlZCgQCUU3/hRdeUH19vfr376/S0lJ997vfjVkH1+kDABBnRUVFKigoiBr/asrvjp/+9Ke69dZbdcYZZ2jTpk26/fbbtXbtWvXr16/T76HpAwBgCMUh6Z9sGb8zHo9HTU1NCoVCcrlcCoVCam5ulsfjiTiuf//+HR9/73vfk8fj0QcffKCxY8d2Oje34QUAwJDo3fuZmZnyer3y+/2SJL/fL6/XG7W039TU1PHxe++9p48//lgXXHBBzLlJ+gAAJJnKykqVlZVp8eLFcrvdqq6uliQVFxdr1qxZGjlypBYsWKDt27fL6XTqjDPO0COPPBKR/k/GEQ733r2H0r47s7eeCgkw8+2NiS4hpmS/I1+y4458SLQvtiztlef5v49aenzOKy/M7PE5TwfL+wAA2ATL+wAAGOJxyV6yoOkDAGCIx+79ZMHyPgAANkHSBwDA0G7doE/SBwDALkj6AAAYQhaO+iR9AABsgqQPAICBS/YAALCJkHV7Psv7AADYBUkfAACDlZf3SfoAANgESR8AAIOVL9mj6QMAYGB5HwAApDySPgAABi7ZAwAAKY+kDwCAwcrn9Gn6AAAY2i28e5/lfQAAbIKkDwCAgY18AAAg5ZH0AQAwWHkjH0kfAACbIOkDAGAIWTjp0/QBADBwyR4AAEh5JH0AAAxcsgcAAFIeSR8AAIOVL9mj6QMAYGD3vg04nK5El9Clm9/ckOgSYlo2OjfRJcTWHkp0BYijcAr8fFPh/zOwNpo+AACGEJfsAQCAVEfSBwDAYOWkT9MHAMBg5abP8j4AADZB0gcAwEDSBwAAKY+kDwCAwcpJn6YPAIDByk2f5X0AAGyCpA8AgIGkDwAAek1jY6NmzJihvLw8zZgxQ7t37+702I8++kiXXnqpqquru5yXpg8AgCHUHu7xR3dVVFSosLBQ69atU2FhocrLy09eayikiooKTZo06ZTmZXkfAIA4CwaDCgaDUeNut1tutztirKWlRQ0NDVq+fLkkyefzad68eWptbVVGRkbEsU8++aS+//3v68iRIzpy5EiXddD0AQAwxOOcfm1trWpqaqLGS0pKVFpaGjEWCASUnZ0tl+vLt2J2uVzKyspSIBCIaPo7duxQfX29VqxYocWLF59SHTR9AAAM8Wj6RUVFKigoiBr/aso/VcePH9eDDz6o3/3udx1/HJwKmj4AAHF2smX8zng8HjU1NSkUCsnlcikUCqm5uVkej6fjmE8//VR79uzRLbfcIunL0wfhcFiHDh3SvHnzOp2bpg8AgOFEgi/Zy8zMlNfrld/vV35+vvx+v7xeb8TSfk5OjjZv3tzx+eOPP64jR47ovvvuizk3u/cBAEgylZWVqqurU15enurq6lRVVSVJKi4u1rZt2057XpI+AACGZLg5z5AhQ7Rq1aqo8aVLl570+K9uBuwMTR8AAEMyNP14YXkfAACbIOkDAGAIhUn6AAAgxZH0AQAwWPmcPk0fAACDlZs+y/sAANgESR8AAANJHwAApDySPgAAhlB7e6JLiBuSPgAANkHSBwDAYOVz+jR9AAAMVm76LO8DAGATJH0AAAwnSPoAACDVkfQBADBY+Zw+TR8AAIOVmz7L+wAA2ARJHwAAA0kfAACkPJI+AAAGKyd9mj4AAAaa/klMmzZNa9as6claEurmNzckuoQuLR8zOdElxNYeSnQFKc3hdCW6BNhcmP+GLS9m0//www87/drBgwd7vBgAABItbNek7/P5NHDgQIXD0f8AbW1tcSsKAAD0vJhNf+DAgfrzn/+s7OzsqK9NmDAhbkUBAJAo7RZO+jEv2ZsyZYo+/vjjk35t8uQkP78MAAAixEz69913X6dfe+CBB3q8GAAAEu1kp7Stgkv2AAAwWHkjH3fkAwDAJkj6AAAYbLuRDwAAWAdJHwAAQ7g90RXED00fAACDlXfvs7wPAIBNkPQBADCwkQ8AAKQ8kj4AAAYr35yHpg8AgMHKTZ/lfQAAbIKkDwCAoZ1L9gAAQKoj6QMAYLDyOX2aPgAASaaxsVFlZWVqa2tTenq6qqurNXjw4Ihjnn32WT311FNyOp1qb2/Xddddp5tuuinmvDR9AAAMyZD0KyoqVFhYqPz8fK1evVrl5eVasWJFxDF5eXn60Y9+JIfDoUOHDmnatGkaO3asRowY0em8NH0AAAzxuCNfMBhUMBiMGne73XK73RFjLS0tamho0PLlyyVJPp9P8+bNU2trqzIyMjqO++Y3v9nx8eeff67jx4/L4XDErIOmDwBAnNXW1qqmpiZqvKSkRKWlpRFjgUBA2dnZcrlckiSXy6WsrCwFAoGIpi9JL7/8shYsWKA9e/bonnvu0fDhw2PWQdMHAMAQj3fZKyoqUkFBQdT4V1N+d02cOFETJ07UJ598ojvuuENXXXWVLrzwwk6Pp+kDABBnJ1vG74zH41FTU5NCoZBcLpdCoZCam5vl8Xg6/Z6cnByNHDlS//jHP2I2fa7TBwDAEG7v+Ud3ZGZmyuv1yu/3S5L8fr+8Xm/U0v6uXbs6Pm5tbdXmzZs1bNiwmHOT9AEAMCTDW+tWVlaqrKxMixcvltvtVnV1tSSpuLhYs2bN0siRI/XMM89o06ZN6tOnj8LhsG688UaNHz8+5rw0fQAAksyQIUO0atWqqPGlS5d2fDx37txuz0vTBwDAkAzX6ccL5/QBALAJkj4AAAYrJ32aPgAABt5aFwAApDySPgAABisv75P0AQCwCZI+AAAGkj4AAEh5JH0AAAzJcBveeKHpAwBgiMdb6yYLlvcBALAJkj4AAAYrb+Tr1aY/8+2Nvfl03bJsdG6iS+haeyjRFSCOwvx8vxaH05XoElIe/4bWR9IHAMDARj4AAGzCyqtubOQDAMAmSPoAABhI+gAAIOWR9AEAMFg56dP0AQAwhEPWbfos7wMAYBMkfQAADFZe3ifpAwBgEyR9AAAMJH0AAJDySPoAABisnPRp+gAAGKzc9FneBwDAJkj6AAAYSPoAACDlkfQBADC0Wzjp0/QBADCwvA8AAFIeSR8AAANJHwAApDySPgAAhnDIukmfpg8AgIHlfQAAkPJI+gAAGEj6AAAg5ZH0AQAwkPQBAEDKI+kDAGAIt7cnuoS4oekDAGBIhuX9xsZGlZWVqa2tTenp6aqurtbgwYMjjlm0aJHWrl0rp9OpM844Q3fffbeuvPLKmPPS9AEASDIVFRUqLCxUfn6+Vq9erfLycq1YsSLimFGjRmnmzJnq27evduzYoRtvvFH19fU688wzO52Xpg8AgCEeST8YDCoYDEaNu91uud3uiLGWlhY1NDRo+fLlkiSfz6d58+aptbVVGRkZHceZqX748OEKh8Nqa2vTgAEDOq2Dpg8AQJzV1taqpqYmarykpESlpaURY4FAQNnZ2XK5XJIkl8ulrKwsBQKBiKZveu6553T++efHbPgSTR8AgAjtcUj6RUVFKigoiBr/aso/Ha+//roWLlyoZcuWdXksTR8AAEM83nDnZMv4nfF4PGpqalIoFJLL5VIoFFJzc7M8Hk/UsW+99ZZ+9atfafHixbrwwgu7nJvr9AEASCKZmZnyer3y+/2SJL/fL6/XG7W0v3XrVt199936wx/+oIsvvviU5qbpAwBgCLeHevzRXZWVlaqrq1NeXp7q6upUVVUlSSouLta2bdskSVVVVfr8889VXl6u/Px85efna+fOnTHndYTD4XD3/0lOz62Owb31VN22bHRuoksA8DU4nK5El4A4+2LL0l55nr7j7ujxOY9uXtTjc54OzukDAGBIhpvzxAtNHwAAg5WbPuf0AQCwCZI+AAAGKyf9Xt3IBwAAEoflfQAAbIKmDwCATdD0AQCwCZo+AAA2QdMHAMAmaPoAANgETR8AAJug6QMAYBM0fQAAbIKmDwCATfRK029sbNSMGTmcn4MAAAPSSURBVDOUl5enGTNmaPfu3b3xtJZw8OBBFRcXKy8vT9OmTVNJSYlaW1sTXVZKqqmp0fDhw/X+++8nupSU88UXX6iiokJTpkzRtGnT9OCDDya6pJTyyiuvaPr06crPz9c111yj9evXJ7ok2FSv3Hv/pptu0rXXXqv8/HytXr1azz77rFasWBHvp7WEtrY27dy5U+PGjZMkVVdX67///a/mz5+f4MpSy/bt2/Xoo4/qo48+0pIlSzRs2LBEl5RSHnroITmdTt1///1yOBw6cOCAzj333ESXlRLC4bDGjh2rlStXatiwYdqxY4euv/56bdmyRU4ni63oXXH/jWtpaVFDQ4N8Pp8kyefzqaGhgbR6itLT0zsaviSNHj1an3zySQIrSj3Hjh3Tb37zG1VWVia6lJR0+PBhPffcc7rzzjvlcDgkiYbfTU6nU5999pkk6bPPPlNWVhYNHwkR97fWDQQCys7OlsvlkiS5XC5lZWUpEAgoIyMj3k9vKe3t7Xr66aeVm5ub6FJSysKFC3XNNddo0KBBiS4lJe3du1fp6emqqanR5s2bdfbZZ+vOO+/U5ZdfnujSUoLD4dBjjz2m22+/XWeddZYOHz6sJ598MtFlwab4UzOFzJs3T2eddZZuvPHGRJeSMt566y29++67KiwsTHQpKSsUCmnv3r266KKL9Ne//lX33nuvSktLdejQoUSXlhJOnDihP/7xj1q8eLFeeeUVPfHEE7rrrrt0+PDhRJcGG4p70/d4PGpqalIoFJL05f9Ampub5fF44v3UllJdXa3//Oc/euyxx1gW7IY33nhDu3bt0sSJE5Wbm6v9+/fr5z//uerr6xNdWsrweDzq06dPxym6Sy+9VP369VNjY2OCK0sN7733npqbm3XZZZdJki677DL17dtXu3btSnBlsKO4d4/MzEx5vV75/X5Jkt/vl9frZWm/GxYsWKB3331XixYtUlpaWqLLSSm33HKL6uvrtXHjRm3cuFEDBgzQn/70J40fPz7RpaWMjIwMjRs3Tps2bZL05dU4LS0t+va3v53gylLDgAEDtH//fn300UeSpF27dqmlpUXnn39+giuDHfXK7v1du3aprKxMwWBQbrdb1dXVuvDCC+P9tJbwwQcfyOfzafDgwTrzzDMlSYMGDdKiRYsSXFlqys3NZff+adi7d6/mzp2rtrY29enTR3fddZcmTJiQ6LJSxvPPP6+lS5d2bIScNWuWJk2alOCqYEe90vQBAEDicXIYAACboOkDAGATNH0AAGyCpg8AgE3Q9AEAsAmaPgAANkHTBwDAJv4fSsJ3HG1rtcQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G0dd2j54Wlaz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "ac685449-2a56-461c-f822-7a61e07ea733"
      },
      "source": [
        "wandb.init(entity='authors', project='loss-landscape', id='resnet20v1_aug_independent_model_investigations')\n",
        "wandb.log({'independent_prediction_disagreement': wandb.Image('independent_prediction_disagreement.png')})\n",
        "wandb.log({'independent_functional_similarity': wandb.Image('independent_functional_similarity.png')})"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/loss-landscape\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/loss-landscape/runs/resnet20v1_aug_independent_model_investigations\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape/runs/resnet20v1_aug_independent_model_investigations</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}