Visualizing_Function_Space_Similarity_SmallCNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Visualizing Function Space Similarity SmallCNN",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNS5vFRWlKYzJCN5v5VMHsi",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "213f0f79039f432e9dc946fc974f7823": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6d715f44e1ca4b0cbe2374d62ce0caeb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cc3a685d68f040c8afccd2b1ae0390c6",
              "IPY_MODEL_d4ed95490ad849ae87d4d28cba95979b"
            ]
          }
        },
        "6d715f44e1ca4b0cbe2374d62ce0caeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cc3a685d68f040c8afccd2b1ae0390c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_69d10af5c7d644868b54ad481d42b856",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_495abdbcc145463a929ccdd1c013a7ef"
          }
        },
        "d4ed95490ad849ae87d4d28cba95979b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8832887c16f94cd7a1453cf06aedd617",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [05:59&lt;00:00,  9.00s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce83b03b47444dfe9d9a6d426026f2d9"
          }
        },
        "69d10af5c7d644868b54ad481d42b856": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "495abdbcc145463a929ccdd1c013a7ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8832887c16f94cd7a1453cf06aedd617": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce83b03b47444dfe9d9a6d426026f2d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8d49a8e92764322bab86026634b84c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f67d1853cd1647bab99c576c40adc2ab",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_484827bb3c264c24953b4975527d175d",
              "IPY_MODEL_898883d27fd54247b7857acca2f9483c"
            ]
          }
        },
        "f67d1853cd1647bab99c576c40adc2ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "484827bb3c264c24953b4975527d175d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_545e5780e80c4ba7b8e0ec7fc2de5e83",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5c79d885d97745879832f7b77d61db3d"
          }
        },
        "898883d27fd54247b7857acca2f9483c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3944b12f65eb4cb49ab47846c91bb8e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [00:14&lt;00:00,  2.79it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_db204126bd8d4f8a87a8816cc164f9d8"
          }
        },
        "545e5780e80c4ba7b8e0ec7fc2de5e83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5c79d885d97745879832f7b77d61db3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3944b12f65eb4cb49ab47846c91bb8e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "db204126bd8d4f8a87a8816cc164f9d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "06aec662cf6948189ee8e371d5287384": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fcf70e91da844425846d47710b3d27d2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0992be684a114c469ceb05371e23c13b",
              "IPY_MODEL_a0fa4a5402214313b515bf32b70a2c20"
            ]
          }
        },
        "fcf70e91da844425846d47710b3d27d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0992be684a114c469ceb05371e23c13b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_edebac964ed940178ac93669431292b5",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4a4c4f42d6a44959b502d94692a3ef56"
          }
        },
        "a0fa4a5402214313b515bf32b70a2c20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_33ddad7f5d53479788a45062f3d5a856",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [13:40&lt;00:00, 20.50s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_edec4ec464d44fe188ebe8b45802e9bd"
          }
        },
        "edebac964ed940178ac93669431292b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4a4c4f42d6a44959b502d94692a3ef56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33ddad7f5d53479788a45062f3d5a856": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "edec4ec464d44fe188ebe8b45802e9bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b7c1be9132046529000e7316f5dcabc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9c9023cf2f554d6eaccc67a22af7e508",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b394ade9ec79466b9eb4e070b42796ba",
              "IPY_MODEL_6a5c2a702644452284c07f0e29577e3d"
            ]
          }
        },
        "9c9023cf2f554d6eaccc67a22af7e508": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b394ade9ec79466b9eb4e070b42796ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_775fb0f3e2584cff9c818ca232559bbf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8e9cb227fbc4ba29cd35b7b3b0538f0"
          }
        },
        "6a5c2a702644452284c07f0e29577e3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1a72d8b8a4a8425a8be59de1ac3995d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40/40 [00:09&lt;00:00,  4.27it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c66e02e3967946869659f7f2e9e0002d"
          }
        },
        "775fb0f3e2584cff9c818ca232559bbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b8e9cb227fbc4ba29cd35b7b3b0538f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a72d8b8a4a8425a8be59de1ac3995d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c66e02e3967946869659f7f2e9e0002d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4637d6fd843045489c4ad210d0ee89e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8ae82964608a44edb8f5e268a4046061",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_184dc95f2fa3415d8b72538ffd11ed0f",
              "IPY_MODEL_bbcdf9fa67e44a798eaa2516914deaee"
            ]
          }
        },
        "8ae82964608a44edb8f5e268a4046061": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "184dc95f2fa3415d8b72538ffd11ed0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_60d2d2edd2944492b89ff6c9662b9fea",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c544d2cfbe76463eb3ece0e3424b6d95"
          }
        },
        "bbcdf9fa67e44a798eaa2516914deaee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_61068162282549e5b5294e30f76e7f84",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [01:02&lt;00:00,  6.25s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a7154471f29417c97196e2ff2601436"
          }
        },
        "60d2d2edd2944492b89ff6c9662b9fea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c544d2cfbe76463eb3ece0e3424b6d95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61068162282549e5b5294e30f76e7f84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a7154471f29417c97196e2ff2601436": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6127015f31134069ac03cfa32aaf292b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b3be6995478148de8f9ebe9cc4e38ab4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_67a8a5c7ff874f11ae52135ff605588d",
              "IPY_MODEL_77800b0a1af04b4681d6422271fde7b2"
            ]
          }
        },
        "b3be6995478148de8f9ebe9cc4e38ab4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "67a8a5c7ff874f11ae52135ff605588d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a695b28df7824d918cf6ba98385c520f",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8ba9e041a5714de3aae213ffd1e429e7"
          }
        },
        "77800b0a1af04b4681d6422271fde7b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9e4500b363d04d5cb2cb4d71971b2501",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:07&lt;00:00,  1.40it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9a0a947f02f4436eb102b2aa12d2d463"
          }
        },
        "a695b28df7824d918cf6ba98385c520f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8ba9e041a5714de3aae213ffd1e429e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e4500b363d04d5cb2cb4d71971b2501": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9a0a947f02f4436eb102b2aa12d2d463": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "218521c71fc3446f84e98a8f4ce58ce3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_506668264965452c928dc5e79f977c10",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_33ad336ed5d24cda83ccefca6ec6fabe",
              "IPY_MODEL_ab14831d6979459fa56f9a5020c87c39"
            ]
          }
        },
        "506668264965452c928dc5e79f977c10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33ad336ed5d24cda83ccefca6ec6fabe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4bd35e14c57943b38c0823c5939e387f",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e71e4325049484bb997aa508bfa251f"
          }
        },
        "ab14831d6979459fa56f9a5020c87c39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_de958c9039cf498892218990dc29139c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:01&lt;00:00,  7.49it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_035e47e2de8345b7935b7c6845803132"
          }
        },
        "4bd35e14c57943b38c0823c5939e387f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e71e4325049484bb997aa508bfa251f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "de958c9039cf498892218990dc29139c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "035e47e2de8345b7935b7c6845803132": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e6352df87c7541d19b45e3726e17d1a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e88a5dda7dcb41469e836908f626b452",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_471d89ea16f04055ba49485307ef9ed8",
              "IPY_MODEL_a713e10e41464f6cb4a3dfecdd7a3eea"
            ]
          }
        },
        "e88a5dda7dcb41469e836908f626b452": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "471d89ea16f04055ba49485307ef9ed8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_752af4f19b1a42e094bbfa539670fb1a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a49bbe7a7a194dc78aaca78e984174a6"
          }
        },
        "a713e10e41464f6cb4a3dfecdd7a3eea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_642bc68664bb4b5ca4d3f7049ac6d204",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [01:45&lt;00:00, 10.52s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03ef3c15f9704cf6bd7fd1df3fe3dfd1"
          }
        },
        "752af4f19b1a42e094bbfa539670fb1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a49bbe7a7a194dc78aaca78e984174a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "642bc68664bb4b5ca4d3f7049ac6d204": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03ef3c15f9704cf6bd7fd1df3fe3dfd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4cfdfaa43b754161878c1a19a774e60a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_68cee03dfe7b4f01ae9819e588a39501",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_26b52483f8d84ea5960eddfc2a23e64e",
              "IPY_MODEL_302f3101dd0746aaabb9513846645a13"
            ]
          }
        },
        "68cee03dfe7b4f01ae9819e588a39501": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "26b52483f8d84ea5960eddfc2a23e64e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ef2ed295bda14b3eba3e463017d45d92",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4a28c983ba824378a6bcf89d81f0b5f3"
          }
        },
        "302f3101dd0746aaabb9513846645a13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_edbcbbe69090421c93ce13603a44b60e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [22:15&lt;00:00, 445.23s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f9f9ea7649064c19af12007ac252cce3"
          }
        },
        "ef2ed295bda14b3eba3e463017d45d92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4a28c983ba824378a6bcf89d81f0b5f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "edbcbbe69090421c93ce13603a44b60e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f9f9ea7649064c19af12007ac252cce3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayulockin/LossLandscape/blob/master/Visualizing_Function_Space_Similarity_SmallCNN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kWXU2ICvCDN1",
        "colab_type": "text"
      },
      "source": [
        "# Setups, Imports and Installations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQAggd-btWJK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        },
        "outputId": "92c02197-e109-419c-d631-854d083d85fc"
      },
      "source": [
        "## This is so that I can save my models.\n",
        "from google.colab import drive\n",
        "drive.mount('gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sAZBTFNQq0ys",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install wandb"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-APP7-6C5uK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.applications import resnet50"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Pr3HrMxq8SF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "5abf9c44-da5d-4398-ef39-5984ac15720c"
      },
      "source": [
        "import os\n",
        "os.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\n",
        "\n",
        "import numpy as np\n",
        "from numpy.linalg import norm\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from mpl_toolkits.mplot3d import Axes3D\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "sns.set()\n",
        "\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "from sklearn.manifold import TSNE"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XmVkQbw_q_07",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "outputId": "97242ddf-27c4-4f93-a534-20ae27b10192"
      },
      "source": [
        "import wandb\n",
        "from wandb.keras import WandbCallback\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://app.wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Not authenticated.  Copy a key from https://app.wandb.ai/authorize\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "API Key: ··········\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FhhwqSIdCu9V",
        "colab_type": "text"
      },
      "source": [
        "# Get Trained Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qDLJ-R6jCz1I",
        "colab_type": "text"
      },
      "source": [
        "(For now I am using models trained from SmallCNN.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WWCudbf8CgoH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ROOT_PATH = 'gdrive/My Drive/LossLandscape/'\n",
        "## SmallCNN model checkpoint for each epoch(total 40 epochs)\n",
        "MODEL_PATH = ROOT_PATH+'SmallCNN_CheckpointID_1/'\n",
        "## SmallCNN models with different initialization(total 10 models)\n",
        "INDEPENDENT_MODEL_PATH = ROOT_PATH+'SmallIndependentSolutions/'\n",
        "## SmallCNN models with different initialization(total 3 models) with all checkpoints saved\n",
        "INDEPENDENT_MODEL_PATH_tsne = ROOT_PATH+'SmallIndependentSolutions_tsne/'"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pZjcYJYJDqHA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "d15c3b1f-80c6-4510-bd13-0681be016a7e"
      },
      "source": [
        "same_model_ckpts = os.listdir(MODEL_PATH)\n",
        "independent_models = os.listdir(INDEPENDENT_MODEL_PATH)\n",
        "independent_models_tsne = os.listdir(INDEPENDENT_MODEL_PATH_tsne)\n",
        "\n",
        "\n",
        "print(len(same_model_ckpts))\n",
        "print(len(independent_models))\n",
        "print(len(independent_models_tsne))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "40\n",
            "10\n",
            "3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c54vIBDCVFoB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# https://stackoverflow.com/a/2669120/7636462\n",
        "import re \n",
        "\n",
        "def sorted_nicely(l): \n",
        "    \"\"\" Sort the given iterable in the way that humans expect.\"\"\" \n",
        "    convert = lambda text: int(text) if text.isdigit() else text \n",
        "    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] \n",
        "    return sorted(l, key = alphanum_key)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mlmgy1YPVJwF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "outputId": "f87e49f1-d0e2-4ed9-8d55-7bfc7404244d"
      },
      "source": [
        "same_model_ckpts = sorted_nicely(same_model_ckpts)\n",
        "same_model_ckpts[:5]"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['small_cnn_checkpoint_0.h5',\n",
              " 'small_cnn_checkpoint_1.h5',\n",
              " 'small_cnn_checkpoint_2.h5',\n",
              " 'small_cnn_checkpoint_3.h5',\n",
              " 'small_cnn_checkpoint_4.h5']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jbb7Tq0yVWSC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "outputId": "6477aab9-c6f9-4188-d5c5-8f355b55412d"
      },
      "source": [
        "independent_models = sorted_nicely(independent_models)\n",
        "independent_models[:5]"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['smallcnn_independent_model_0.h5',\n",
              " 'smallcnn_independent_model_1.h5',\n",
              " 'smallcnn_independent_model_2.h5',\n",
              " 'smallcnn_independent_model_3.h5',\n",
              " 'smallcnn_independent_model_4.h5']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fzS6bWLyEwU2",
        "colab_type": "text"
      },
      "source": [
        "# Get Dataset and Prepare"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMFYLweasySz",
        "colab_type": "text"
      },
      "source": [
        "#### CIFAR-10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Posjlu7lDJBd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "53a011eb-8543-4b81-97e5-2f1bbb048c51"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
        "\n",
        "y_train = y_train.flatten()\n",
        "y_test = y_test.flatten()\n",
        "\n",
        "CLASS_NAMES = (\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\")\n",
        "print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 11s 0us/step\n",
            "(50000, 32, 32, 3) (50000,) (10000, 32, 32, 3) (10000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QZzY01xwxS-f",
        "colab_type": "text"
      },
      "source": [
        "#### Dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6TqcgM67xxDF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "BATCH_SIZE = 128\n",
        "IMG_SHAPE = 32\n",
        "\n",
        "testloader = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
        "\n",
        "def preprocess_image(image, label):\n",
        "  img = tf.cast(image, tf.float32)\n",
        "  img = img/255.\n",
        "\n",
        "  return img, label\n",
        "\n",
        "testloader = (\n",
        "    testloader\n",
        "    .map(preprocess_image, num_parallel_calls=AUTO)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(AUTO)\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KRQ8bw4QD6Xk",
        "colab_type": "text"
      },
      "source": [
        "#   Similarity of Functions Within Initialized Trajectories"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12zzFXslEOUR",
        "colab_type": "text"
      },
      "source": [
        "## Disagreement of predictions \n",
        "\n",
        "(Refer figure 2(b) from paper)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "daTRJDfJJHJ1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Given model and test data, return true_labels and predictions.\n",
        "def evaluate(test_dataloader, model):\n",
        "  true_labels = []\n",
        "  pred_labels = []\n",
        "  \n",
        "  for imgs, labels in iter(test_dataloader):\n",
        "    preds = model.predict(imgs)\n",
        "    \n",
        "    true_labels.extend(labels)\n",
        "    pred_labels.extend(np.argmax(preds, axis=1))\n",
        "\n",
        "  return np.array(true_labels), np.array(pred_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hCM_JKfoEoa2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "213f0f79039f432e9dc946fc974f7823",
            "6d715f44e1ca4b0cbe2374d62ce0caeb",
            "cc3a685d68f040c8afccd2b1ae0390c6",
            "d4ed95490ad849ae87d4d28cba95979b",
            "69d10af5c7d644868b54ad481d42b856",
            "495abdbcc145463a929ccdd1c013a7ef",
            "8832887c16f94cd7a1453cf06aedd617",
            "ce83b03b47444dfe9d9a6d426026f2d9"
          ]
        },
        "outputId": "461622e5-70a3-4175-f6d2-d6d9ec266091"
      },
      "source": [
        "predictions = []\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model(MODEL_PATH+same_model_ckpts[i])\n",
        "  # get predictions for model\n",
        "  _, preds = evaluate(testloader, model)\n",
        "\n",
        "  predictions.append(preds)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "213f0f79039f432e9dc946fc974f7823",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4axScVEyIDbJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "b8d49a8e92764322bab86026634b84c4",
            "f67d1853cd1647bab99c576c40adc2ab",
            "484827bb3c264c24953b4975527d175d",
            "898883d27fd54247b7857acca2f9483c",
            "545e5780e80c4ba7b8e0ec7fc2de5e83",
            "5c79d885d97745879832f7b77d61db3d",
            "3944b12f65eb4cb49ab47846c91bb8e7",
            "db204126bd8d4f8a87a8816cc164f9d8"
          ]
        },
        "outputId": "ec437834-2ec2-45a3-f5ef-e246a8f1ee15"
      },
      "source": [
        "empty_arr = np.zeros(shape=(40,40))\n",
        "\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  preds1 = predictions[i]\n",
        "  for j in range(i, 40):\n",
        "    preds2 = predictions[j]\n",
        "    \n",
        "    # compute dissimilarity\n",
        "    dissimilarity_score = 1-np.sum(np.equal(preds1, preds2))/10000 \n",
        "    \n",
        "    empty_arr[i][j] = dissimilarity_score\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = dissimilarity_score\n",
        "\n",
        "dissimilarity_coeff = empty_arr[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8d49a8e92764322bab86026634b84c4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0yDZamrkVbXb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 485
        },
        "outputId": "855cdb7d-fa79-42d2-a4ac-a2d0dfc1a5af"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns.heatmap(dissimilarity_coeff, cmap='RdBu_r');\n",
        "plt.xticks([5,10,15,20,25,30,35],[5,10,15,20,25,30,35]);\n",
        "plt.yticks([5,10,15,20,25,30,35],[35,30,25,20,15,10,5]);\n",
        "\n",
        "plt.savefig('prediction_disagreement.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAHUCAYAAAAwb/F2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3Rb9Z33+68ky7JsR77FdhwnJCQkYMqldGjpDAVabuGcY3LheVjhZOjQ1UPSFhI6vZCkHYhJSKeEYcFASJ4+wxQaVsqUE5jh4tLSoQynJdMyXAKFmtAQco9jJ5Zvsi3Lkvb5gyHdxon8MZEdS3q/1vJaQf10a++tLf3029r6yOM4jmMAACDreU/2CgAAgLHBoA8AQI5g0AcAIEcw6AMAkCMY9AEAyBF5Y3lnt/hOHTbzP8+vkZY1+bNTpNyU+sulXP5p50i5RHGllEsWlkk5M7M+b4GUa48mpFxnv5YL9w1IOZ/HI+VaemJSriuq3e8RcXnhiJarLQ9KuVg8KeW2N3dLuYi4vfsPRaRcsDhfyjlJ7Ys5Rw5q2xGPacdVXr5Pynl92nHV29Uv5TxebXnRnqiUMzNL9PdJuZLqiVJuoD8u5aI9vVIu3qcdM3nBYinnzdOOrb62A1KuZPI0Kbfr5WekXGzbw1LuRH3dMz3ty/yRszvty/wkmOkDAJAjGPQBAMgRY3p6HwCA8U785CkjMegDAOCiXseUiTi9DwBAjmCmDwCASzaf3memDwBAjmCmDwCASzZ/ps+gDwCAC6f3AQBAxhvTmX5MqAVVqzQTMa3O0rxaJaiTp1Xhege0ekwnFpByZmYxv5Yt8mvv0bpjWo3sQEKraZ1Uoq3fznZt3/h92nbUhLTHJCjWvqp1vZUhbXtnVBZJuc5erYa3pV2rfFWrXMtKtP2XqNa2o72lR8oVivvPKz7X8wPay1RU3M8jqeGdeEq1lCsu1fb1B2/skHKF5Vrdd3HpBClXXqPV8O75434pVzFthpTb8R//JuVO/cJcKTdWsvn0PjN9AAByBJ/pAwDgwmf6AAAg4zHTBwDAJZs/02fQBwDAJZtPgWfztgEAABdm+gAAuGTz6X1m+gAA5Ahm+gAAuGTzV/YY9AEAcMnm0/tjOugX56Xv04T8kFYdqvLEtIrRRMlkbYF+rZbTzCxffFsZjiakXO+AlvOL99sm1puWiHWp+7u0GlS1vjYS1Wppa8uDUu5Qh7Z+e9q0Y6a1q1/K9fdp2xEIavu5o1u738P7uqRc0tFqmz0RKWbRHu3xVau5k3GtfjoZ1+qYzcwONL0r5cqmaLW0+RPKpVyk9YCUS4jb0hs5Rcp58/KlnFqvO+tLC6Rc254PpBxOHDN9AABcsvn0PhfyAQCQI5jpAwDgwmf6AADkCE7vAwCAjMdMHwAAl2w+vc9MHwCAHMFMHwAAl2z+TJ9BHwAAl2we9Dm9DwBAjhjTmX5ErMlUJKJaxajM65NinoFeKecE9JrgaEKrN80X60jVi1CS2t1aRaFfyh3o1uprVdUhrcrY503vsZAQd8ys6glSrkOsE3bE++0Ra31nTCuVcmr9b8dhrXZYXV5BkXZcqcvzijXfgaJiKTeSbEGRVl/beXCPlFPrcNX639JK7fVo27/9i5Sb/BdzpFx3uEPKBStqpdxY4UI+AACQ8fhMHwAAFz7TBwAAGY+ZPgAALtn8mT6DPgAALpzeBwAAGY+ZPgAALtl8ep+ZPgAAOYKZPgAALtn8mf6YDvol/uFPLCQG0tfaZ2ZmyYQUc6Ja25gTmiTlPLFeSxZVSNmQ2Aa4tysm5fziEVsS0B7+98NaC+HEQq1FrHdAe0z2tGn329ypNQF+cdZEKXdYbLz7zXuHpZwqz68dB7Oma017+8QGva4j2n4uqSiUch6xOVKlNvKp8vJ95vNpJzkjHdo+jIvPpWnnninleiPacz3ao+XUpr3zFvzfUq51f6eUm/apKVJu77vNUm6scHofI6YO+ADGljrgA9mI0/sAALh4mekDAIBMx0wfAAAXTxZfycegDwCAizeLB31O7wMAkCOY6QMA4OLJ4m94ZO+WAQCAQZjpAwDgks0X8jHTBwAgR6Sc6W/dutUuvPBCMzPr7u62NWvW2LZt26yurs4aGhps4kSt1vQjSrWhT6jqNTPzFQSknCfPr+WCE6Sc4xVPjiQGtJyZRZNa/WqeWG/aGdVqS7tjWh1uQPx8q7lbq6/t6NP2TZ+4fsViBerejj4pl5+nbW9xgXa/6nY4jiPlDoa17YjHtErrWL92vHSKtcjFpQVSrr0lIuWCxdpzPSYeV5EjYSlnZpaIafu6oEjb5rZD3VKur/2IlDu8/fdSbtpfXS3lOsTq5p7De6VcWHzs4n3asTBWcvbq/Xvuuefov++77z4rKiqyjRs32owZM2zt2rWjvnIAAIw1j9eb9r/xIuVUxT3zeP311+2JJ54wv99vs2fPtquv1t45AgCA8SHloB+LxWznzp3mOI55PB7z+/98qtw7jt65AACQLtl8ej/loB+NRm3x4sVH/7ulpcWqq6stEokw6AMAkGFSDvovvvjikNs6OzstEAjYAw88MGorBQDAyZKzX9nbvn27XXPNNXbttdfazp07bcmSJXbxxRfbnDlzrKdHu8oTAIBM4vF50/43XqRck7Vr19rNN99sf/3Xf2033nij1dfX21tvvWUNDQ121113jdU6AgCANEg56Pf09Nhll11m8+fPNzOzuXPnmpnZpZdeah0dHaO/dgAAjDGvz5P2v/FC/sreRyU9H0kmteIPAAAwMrt27bKVK1daR0eHlZaW2rp162z69OmDMuvXr7fHHnvMqqqqzMzsM5/5jDU0NKRcbspBv7a21iKRiBUXFw8q4zl06JAFg8FPuCkAAIxfHrH9dDQ1NDTYokWLbN68efb000/bqlWr7NFHHx2Smz9/vq1YsUJebspBf8OGDce8PRQK2caNG+U7+UhfYvizA17xggf5wgivVnFrHvF+k1plqVao+qH+hJYWW1otIQYHhMfDzKw8mC/lxJZWC0diUk6tue3o1epX1XrdD8Qq0gqxYvRP+7SPwtQ63H6xbra0skjK5Ys1xr0RrWZZ5SS147RPvN9EXDuePT7xNcHMPOLrR3xAq1qOdrZLObVet/KMz0u5giLxObxHq9f15WnLU4/pvGCxlBsr6jg0El1dXdbV1TXk9lAoZKFQaNBtbW1t1tTUZI888oiZmdXX19udd95p4XDYysvLT2g9PtGv7BUWFlphYeEJ3TEAALli06ZN9uCDDw65fenSpbZs2bJBtzU3N1t1dbX5/vsNqs/ns6qqKmtubh4y6P/85z+3l19+2SorK23ZsmV23nnnpVwPfloXAACX0fie/g033GALFiwYcvvHZ/kjcd1119nXv/518/v9tnXrVrvpppvsueees7KysuP+fxj0AQAYZcc6jX88NTU11tLSYolEwnw+nyUSCWttbbWamppBucrKyqP/vvDCC62mpsZ27Nhhn/vc54677PHTGAAAwDjg8XnS/jcSFRUVVldXZ42NjWZm1tjYaHV1dUNO7be0tBz997vvvmsHDhywU089NeWymekDADDO3HHHHbZy5UrbuHGjhUIhW7dunZmZLV682G655RY7++yz7d5777U//vGP5vV6ze/329133z1o9n8sDPoAALiMxtX7IzVz5kzbsmXLkNsfeuiho//+6I3ASDDoAwDgkrM/uAMAALIHM30AAFy846CRb7Qw0wcAIEeM6UxfaZtVP0tJRMVKULFG0/Fqu8IT1+7X4+g/SCS2kZrYIms+j7YP/eLFKoV+bR92x7Qq0pJCv5QL5ut1qYo+cf2KxVpa1WSxDnfP/qEVncfieLQDRq2GLS4tkHIJsba5KKTVEweC4nNOPJ77xHrnnr6IlDMzm1BVLeXUffiH3zVKuaozLxw+ZGYFJccvYXErLNZqcw/HtX0YrKiVcr3hw1KueGKVlBsrcs17BuL0PgAALuPpp3DTLXvfzgAAgEGY6QMA4MJX9gAAQMZjpg8AgAsX8gEAkCO4kA8AAGQ8ZvoAALh4aOQDAACZjpk+AAAu4+GndUfLmA76yrUR0faotCyPV3tQkl1tUs4iWs6pnqXdb75WvWpmViw29u7v0mpVVQVir++R3gEpN6uiUMq92azVzXaK97ujpVvKXX3OZCm3p61VyvXF4lLuTzu0YytPrB0OFGg1xtOrJ0i5V/bul3Jqve5Av3acxsVa5P6odhw4Yp/1pNO048DMrHXPESn3wW+elnLnzF0o5ToO90i5PLEiO9Khva5Wz9Je36I9Wl3vFHF57S3a9uLEMdMHAMAlm8t5GPQBAHDJ5u/pZ++WAQCAQZjpAwDgol4zlomyd8sAAMAgzPQBAHDhK3sAAOQILuQDAAAZj5k+AAAuzPQBAEDGG+Ma3uFbjvKC2ip587UqUk+BVoer5hyfuMscsVvXzGIJrf1J2H1mZjYg1pFG4/o6KnoHtFpVtV63LaJVfcbE7dhxOCLlfOIvbLV29Uu5gqJ8KadWparVq7vFeuKe9k4pN9CvPUcKCrXtTTracRqPqTW82vEXbpZiZmbWvO0FKVdz3uVSTn2M+9q1+l91m/MnlEs5f0B77DoP7pJyhaE6bXkt2vaOlWz+yh6n9wEAcPH4tDfWmSh7384AAIBBmOkDAODChXwAACDjMdMHAMDFm8UX8mXvlgEAgEGY6QMA4JLNn+kz6AMA4JLNg372bhkAABhEmum3t7fboUOHzMxs0qRJVlZWNqorBQDAyZKzjXx79+6122+/3ZqamqyqqsrMzFpbW+3MM8+01atX2/Tp09O+Qv4irV43EdUqUL1FE6ScExDrehNxMadVh5qZ9ce1T1n8Yj2sGLP+uFbhWVKgPSZ/OpLemtvSQu1+8/O0J6ha61sa1O63uEB73Eona8fgHrE+Oc+vbW9ft7a9oYnam/iBfu3YLwoFpFxUrGMOBLX919OpvSbs+c9npZyZ2dQL/i8pV16jreOh9w9KOV8gKOWchPYcLi7VXt/6+7THpKCkUsq1t3RJOXV7ceJSvmotX77cFi1aZI888sjRrzAkk0l79tlnbcWKFfb444+PyUoCADBWcvYz/Y6ODps7d+6g7yx6vV6bN2+edXZqP9IBAEAm8fi8af8bL1KuSWlpqTU2Nprj+jUsx3HsmWeesVAoNOorBwAA0ifl6f277rrLGhoabM2aNVZdXW2O41hLS4vV1dXZXXfdNVbrCADAmPGOo5l5uqUc9KdPn26bNm2ycDhszc0f/gh1TU2NlZdrv80MAADGj5RvZ7Zv327XXHONLVmyxAoKCuz++++3L33pS3bJJZfYu+++O1brCADAmPF4vWn/Gy9SrsnatWvt5ptvtuuvv95uvPFGq6+vt7feessaGhps3bp1Y7WOAACMmZy9kK+np8cuu+wymz9/vpmZzZ0718zMLr30Uuvo6Bj9tQMAAGmT8jN991X7F1544aD/LZlMjs4aAQBwEo2nmXm6pdyy2tpai0Q+bFlbu3bt0dsPHTpkwSANSgAAZJKUM/0NGzYc8/ZQKGQbN24clRWK92lVnypnQKzDjWs5Jy9fyg34tCpSMzNfQjtr0jOg7ZveAW15Xo9Wh9st1q9OKta2+bVwn5QLiPW6HxzW6n8/P7NCyr26KyzlIlFtv0S6tHrYzrZeKafW3Kq5cIu2/7xifbJar+sP+KTc4X1tUq552wtSbtpfXS3lzMxifdqxGm7WlldUrlUeq3W4+QGtCjohvsbk5WuPider1Q4XFGmvl+r2jpXxdOFdun2in9YtLCy0wsLCdK8LAAAYRZ9o0AcAIFt5fdoZj0zEoA8AgEvOXsgHAACyBzN9AABcmOkDAICMx0wfAAAXvrIHAECO4PQ+AADIeGM600+4uvxPlHz6JR7TlueIvyWQSG9joJlZIqntF7FUy3xacZr1ivfrF5vY8sR3x2rTXn9c2+CYmAuKbWPq+lmB9vQJH+6Rcj5x/zni4zapQivQatnbmdb7DR9olXKlNZVSTm3aqznvcikX6+uzskmlUnagX3z9EJ8jwWKtoS4+kJBy+UF/Wpen8orPkYJCbf2iPdp+HivM9AEgS6gDPpCN+EwfAAAXLuQDACBHeLzZW8ObvW9nAADAIMz0AQBwY6YPAAAyHYM+AABuXm/6/0Zo165dtnDhQpszZ44tXLjQdu/efdzsBx98YOeee66tW7du+E0b8ZoAAIBR1dDQYIsWLbLnn3/eFi1aZKtWrTpmLpFIWENDg11+udZTwWf6AAC4eHzp/0y/q6vLurq6htweCoUsFAoNuq2trc2amprskUceMTOz+vp6u/POOy0cDlt5efmg7D/90z/ZF7/4Revt7bXe3t5h14NBHwAAt1G4kG/Tpk324IMPDrl96dKltmzZskG3NTc3W3V1tfn++82Hz+ezqqoqa25uHjTob9++3V5++WV79NFHbePGjdJ6jOmgXyxUN+YFtVUKVpVpd5qn1V46HvGTDnF5fmckdb1ahadf7NcdEOtS42Kvb0lA2+ZWsUrTJ1aWqvXEMyqLpVy+WK1ZXhSQcuGefimX59deQMqqi6ScT6xAjas1ywHtOadWuVZMrZJyTb94QsqdfsX/kHIB8bVjoF+vpA1OCEo5r0c7ph2xilytPO6LaMdg5ZSStC4v2jMg5fLE6uvCkPacy2Q33HCDLViwYMjtH5/lqwYGBuz222+3H/7wh0ffHCiY6QMA4DYKM/1jncY/npqaGmtpabFEImE+n88SiYS1trZaTU3N0czhw4dt7969tmTJEjP78OMDx3EsEonYnXfeedxlM+gDADCOVFRUWF1dnTU2Ntq8efOssbHR6urqBp3anzx5sr3yyitH/3v9+vXW29trK1asSLlsrt4HAMDF4/Wm/W+k7rjjDtu8ebPNmTPHNm/ebKtXrzYzs8WLF9vbb7/9ibeNmT4AAG7joJFv5syZtmXLliG3P/TQQ8fMf/xiwONhpg8AQI5gpg8AgNs4mOmPFmb6AADkCGb6AAC4fJIL7zIFgz4AAG6c3gcAAJluTGf64djw9ZfJmFYN298R0e40rlXDehztfj2x4X/QwMwsHtCrFaMJrdJSbPqUJbSmTysQa1/FlmDrFY4DM7PSQr+U29OmPSYJsQK1s087ZionFEi5D5KdUq67vU/KxcXnyFmnaFXV+4u1muWOwz1STq3XPfP/+J9Srm1fq5Qrr9XqfwvE48rMLBHX9rVajaxWMucHtXUsLtWOQVV/n1YfXlCkrV9SrBNOqi9GY4WZPgAAyHR8pg8AgMto/LTueMFMHwCAHMFMHwAAN76yBwBAjuBCPgAAkOmY6QMA4OJhpg8AADIdM30AANy4kA8AgNyQzaf3Uw76O3bssFmzZpmZ2cDAgG3cuNHeeOMNq6urs29+85sWDAZHdGfl+cPvyEBIqwTNK9By3glaFWkyWKLlCrR63aRY+WpmFherKmNxLRfp16o0w2LdbKFfe9e7t0OrkQ33aPfb2afVE/fFtO39wz6tDld1qFPbXrV6NSE+vvEBrcb47d1hKRfpiEq5XS8/I+VO/cJcKadWvhaWlkq5qHhcqTkzs0RCq+EtqSiUcuq+7hePfZW6fir1sRPbzS0uVnPjxKV8NV++fPnRf69fv97ee+89+8pXvmIdHR3293//96O+cgAAjDmvL/1/40TKmb7jmq3+9re/tc2bN1tRUZFddNFFNn/+/FFfOQAAkD7DDvrRaNQcxzGfz2dFRUUf/p/y8iwvj8sBAABZKFcv5HvvvffsvPPOM8dxzOPxWEtLi1VXV1t/f78lk+KHNQAAZJBs/sGdlIP+9u3bj3l7f3+/rVmzZlRWCAAAjI6U5zDa29vttttus69+9av205/+9OjtoVDIfvzjH4/6ygEAMOay+EK+lIN+Q0ODhUIhu+666+yFF16wpUuXWjz+4Vc19u/fPyYrCAAA0iPloL97925bvny5XXnllfbwww9bZWWlfe1rX7P+/v5BV/YDAJA1cnWmPzDw54IIj8djDQ0NNnv2bFuyZIn19/eP+soBADDWPF5v2v/Gi5RrMnXqVHv11VcH3bZixQo799xzbffu3aO5XgAAIM1SXr1/9913m8fjGXL7t7/9bZs7V6vadPMdY1kf19+lV2QqnLi2PG+sV1te6RQpp1bmmuk1ty0RbVsS4l1PCGhdC80R7azOxCKtGjk/T9veD1ojUq5P3C/TPlUt5SJRrQL1/Z1aza1a5Tog1idXn6JVRscHtPvd8R//JuVmfWmBlCsuLZByveLjNiDWVCfi2vYGgn4pZ2bWd6RdyvWKz6V01/XGxGOmO6xVRpdWFkm5riPa62VZtba89pYeKTdmxtHp+HRLeaSWpui8Pu2009K+MgAAYPRQqwcAgJtn/HwGn27Zu2UAAGAQZvoAALhl8UyfQR8AABcniwf97N0yAAAwCDN9AADcmOkDAIBMx0wfAAA3oUguUzHoAwDgNo668tNtTAf9It/wOzIQ0qpcB3q0mkpfSYWUSxaEpJwnqdVexpP6QZMQa0ZLCrSHK9Cj3XdA/HRnQOz13d2mVXNWTQhIub6Ytq+DVcVSrlusLJ1RqS1vb7NWE3zRWVr97+/+dFjKza7Vanh/8sMHpNxnF14v5fLEuuizp5dLueZOrRo2z6vNug6Jx1+0R6tZNjObWHuKlJtePUHK7W7plnITyoNSTt2WvHztsasu0+63Jai9FpWKz3WfMDYgPZjpAwDgwlf2AABAxmOmDwCAWxbP9Bn0AQBwy+JBP3u3DAAADMJMHwAAN2b6AAAg0zHTBwDAha/sAQCAjMdMHwAAtyye6Y/poH8wOnxl5EBUq0rNK9DqehPtWrWpt3yKlPPE+6VcnrdQypmZRQaSUi6hxay9T6vmbO+NSbnzp5RKuYGktoL/8W6rlJstVpv+Saw2nVZRJOXe2NMu5f7qzCop95u3D0m5cItW6/ufj26Scl/53i1S7r19HVKuZW+nlGsT64kn1mqPb3dYq+v1iHW9wWLttcPMrL9Pez16f5+2bwbEKuiE+GQvqdBeZyaLtb7bd4alXFFIq9dt79Tq0v2BcTb/zOIf3MnetzMAAGCQcfb2CgCAkyyLT+9n75YBAIBBmOkDAOCSzV/ZY9AHAMDNm72DfvZuGQAAGISZPgAAbll8ej97twwAAAzCTB8AALcsnul7HMdxxurOvuufIeWu+SutHW/y50+VcrXXLJBynql1Ui4RmiTlwnHtPVVnf0LKtUS0Br3tR3qkXDSu3e8EsS2rQ2wC/M17WkuiT2xY27+/S8qdNXuitrxwr5SLxbT9FxFbyd566mdSbuYX50s5M7PqU0qk3KHdWiuf2pzmy9MeuwllWlNce4t2TKe7ya6nS2vgLCjySzmfTxtM1O3IFx8Pj9gwFx/Qjmm1/TAQ1NZvQHwN3LryUil3omKdR9K+zPwS7fVntI27tzO5NuADo4UB/9jUAR/IRoxMAAC4ZPP39LN3ywAAwCDM9AEAcBsHM/1du3bZypUrraOjw0pLS23dunU2ffr0QZknn3zSfvKTn5jX67VkMmnXXnut/c3f/E3K5TLoAwAwzjQ0NNiiRYts3rx59vTTT9uqVavs0UcfHZSZM2eOXXPNNebxeCwSidjVV19tn/vc5+yMM8447nJP/tsZAADGE48n/X8j0NbWZk1NTVZfX29mZvX19dbU1GThcHhQrri4+Og3M6LRqA0MDAz7TQ1m+gAAuI3C6f2uri7r6hr69eJQKGShUGjQbc3NzVZdXW0+n8/MzHw+n1VVVVlzc7OVl5cPyv7617+2e++91/bu3Wvf+c537PTTT0+5Hgz6AACMsk2bNtmDDz445PalS5fasmXLPvFyL7vsMrvsssvs4MGDdvPNN9vFF19sM2YcvxOHQR8AAJfR+MreDTfcYAsWDO2N+fgs38yspqbGWlpaLJFImM/ns0QiYa2trVZTU3Pc5U+ePNnOPvtse+mll1IO+nymDwDAKAuFQjZlypQhf8ca9CsqKqyurs4aGxvNzKyxsdHq6uqGnNrfuXPn0X+Hw2F75ZVXbPbs2SnXQ57pf/RZxLFWUJUvVDfGerQqV39hgZRL9mltXv7edikXLztFu98BrR3MzCyYp733OtCl1bn6xGtGKgvzpdxAUmtq/nSNdmxs26Pt6/w8n5RrEZvi6s/S2hTvfqZJyk2cqDW7bd30Eyl37vzrpJxavfqXsyul3C8Oa8+R4hKtQU+tco2Lz5HgBO04VfeLmdmUScVSbu/BbikX7dEqsmfMKpVyB8XHpLBI2zcJ8TkczNdeV2Nx7bE7d6q2vXvatO0dM+PgK3t33HGHrVy50jZu3GihUMjWrVtnZmaLFy+2W265xc4++2x7/PHHbevWrZaXl2eO49j1119vX/jCF1IuN+WzJBwO2z333GO/+MUvzMzMcRzzer121VVX2Xe/+90h7zoAYLxTB3zkLmeEV9uPhpkzZ9qWLVuG3P7QQw8d/ff3v//9ES835duZW2+91aZOnWovvviibdu2zd5880174YUXbMqUKXbrrbeO+M4AAMDJk3LQP3DggH3jG9+wsrKyo7eVl5fbTTfdZPv37x/1lQMAYKw5Tvr/xouUg34gELBt27YNuf2NN96w/HztsyQAADA+pPxMf/Xq1bZ8+XILBAJWW1trZh/O/vv7+49eVAAAQDZJjqepeZqlHPQ//elP2/PPP2/vvPOONTc3m9mH3x8866yzhq36AwAgE2XvkD/M6f329na7/fbb7b777rPW1la78sor7eyzzzaPx3NCDUIAAGDspRz0GxoaLBQK2XXXXWe//vWvbenSpRaPx83MbN++fWOyggAAjKWkk/6/8SLloL97925bvny5XXnllfbwww9bZWWlfe1rX7P+/v6xWj8AAJAmKQf9gYE/t+N5PB5raGiw2bNn25IlSxj4AQBZyXGctP+NFykv5Js6daq9+uqr9tnPfvbobStWrLB77713UCuQqlqoySycqFV9enxaTWLeRK16NVlUIeU6Ylr9ZJ9YUzkSCfG46RFrUNXlvbzjsJT7wiyt9tXn1R67N//YIuXy/Fpd7+92hYcPmdmnZ0+Ucj/+wQSMHkYAAB6USURBVP1Sbs43Fku5SK9WQf3Zmdqx+tt3tP3nE59LtdVFUm6SWNd7uFurle4Xn0uRaFzKxUdwrjVUqtXSnnZKiZSrnKAtr7jAL+XaItrkS903nz1Va1nd0aLVE5cUattR3KPlcOJSjsJ33333Ma/S//a3v21z584dtZUCAOBkGU+fwadbykG/tPT4P5Zw2mmnpX1lAAA42bJ4zOendQEAyBX6b1ECAJADsvn0PjN9AAByBDN9AABcxtNX7NKNQR8AAJf0f+F6/OD0PgAAOYKZPgAALll8dp+ZPgAAuWJMZ/qdQj1sIBSQlpVXkC/lPAVadaiT1Goqi/3a+6Tufv1TIbU2NxLT1jEm1pZWFmr78MoztSrjsmB6q0Ov/PwpUu6NPe1S7vxpZVLu+htuk3L/z999U8q9+acjUk6tfH3nQKeU++I52uP2weGIlPN5h7ZzHku4R3t8y4u057q6PCvQXs5mV0/QlmdmHX1aNXIizd/xUvf1Z8RjWn1N6Itpr0XTKrTXVbWG94wa/TEZC9n8lT1O7wMA4JLNV+9zeh8AgBzBTB8AABe+sgcAADIeM30AAFyy+CN9ZvoAAOQKZvoAALgks3iqz6APAIBL9g75nN4HACBnMNMHAMCFRr408XmGr5b0iTW3Az1RKefExRrNkslSTq3HrCz0STkzs4MtWs3oJdPLpdyrYk3rK7u1+tqLT6uQcv/7tx9IuRmVxVKutVvbL+dOLZVyar3u5k1rpdxrYv3v8rlnSrnGdw5JuRK17rgnJuWmlBVKObWS9pypJVJOeT0wM4sl0lsh29ol1vqa2ZSyoJSbJR7TO8TK4ynl2v2q9bqnTdRqcw/3aseM6rC4r8uLtUpwnDhm+gAAuGTxdXwM+gAAuCWz+FI+LuQDACBHMNMHAMAlm0/vM9MHACBHMNMHAMCFr+wBAJAjOL0PAAAyHjN9AABc+MoeAADIeONupt/T2ivlymZry3P6eqSctycs5Y7kV0m5PZ1aTbCZWaFfq+x95t0WKdfcod33hALt4X/s1X1S7v88p0bKPfvmQSl3uEWrLH3rqZ9Jub9d8x0p9+OXd0m5jiPasfqfTa1SbqA/LuWmTAlJuYrigJTb+kftuJoySauafW2XVk9cHNCOvw/E6tr8PG0Oo9ZAm5n9YV+HlNve3CXlZlVPkHL7w31STvXBYe11cFqFVsnc2atVMteKdcIH0ry9J4rP9AEAQMYbdzN9AABOpmQWT/UZ9AEAcBF/2DEjcXofAIAcwUwfAACXbD69z0wfAIAcwUwfAACXRBbP9Bn0AQBw4fQ+AADIeMz0AQBwyeav7I3poF8bHP7uJtRoFZmhU7XKV0+5lksUT5Ry4S6tKtXv1U+idMe0ZSbEH3kO9/RLuc4+rUoz3KUt70C7VqXZ0xOTcmq97rnzr5NybeL9fmZamZT7VXO3lDtrtnZsdfRq6zdbrXJt12qCg8X5Uq64wC/lppRp1asdYpXrGTVa7XBArOH1eT1SzsysckKBlKsp1XIlhdo+VEWi2mtHTYm2fhPF6mb1tahUPGb6ihNSDicu5bNk69atR//d3d1tt956q11++eW2bNkyO3LkyKivHAAAYy3pOGn/Gy9SDvr33HPP0X/fd999VlRUZBs3brQZM2bY2rVrR33lAAAYawnHSfvfeJHyfLvjWtHXX3/dnnjiCfP7/TZ79my7+uqrR33lAABA+qQc9GOxmO3cudMcxzGPx2N+/58/n/GO4DNrAAAyhXjJQkZKOehHo1FbsmTJ0Rl/S0uLVVdXWyQSYdAHACDDpBz0X3zxxWPe7vP5bP369aOyQgAAnEzqtxMy0SeargeDQfvGN76R7nUBAACjKOVM//333z/u/9be3p72lQEA4GQbT1+xS7eUg359fb3V1tYOuor/Ix0dHaO2UgAAnCyJ7B3zUw/6tbW19thjj1l1dfWQ/+2SSy4ZtZUCAADpl3LQv/LKK+3AgQPHHPSvuOKKEd9Zn/D2qaBCqxj1BbS6yGSBVuGZ8Gp1kcE8rTq0J6bXSn4Q1upS1YtL9h3ukXKFQW2bv/SpoY//saiVoL/b/KiU+8vr/0bKFRWJNbIBrXX6nQOdUu7suiopF4trx4L6+P6pRav/rRPra4P52n6JxbVC8m7xOJhQoN2vul/6xfUrHUEVboVYURzM90m5PvF1Qd3Xk8T631Lxud7drz126a4Tnig+h8dKzp7eX7FixXH/t9tuuy3tKwMAAMx27dplK1eutI6ODistLbV169bZ9OnTB2U2bNhgzz33nHm9XvP7/fatb33LLrroopTL5Vf2AABwGQ9f2WtoaLBFixbZvHnz7Omnn7ZVq1bZo48OPkt6zjnn2Fe/+lULBoO2fft2u/766+3ll1+2goLjnwGiYQcAAJeT/YM7bW1t1tTUZPX19Wb24UX1TU1NFg6HB+UuuugiCwY//FXL008/3RzHGfYie2b6AACMsq6uLuvq6hpyeygUslBo8PU3zc3NVl1dbT7fh9eK+Hw+q6qqsubmZisvLz/m8p966ik75ZRTbNKkSSnXg0EfAACX0fjK3qZNm+zBBx8ccvvSpUtt2bJlJ7Ts//qv/7L777/fHn744WGzDPoAAIyyG264wRYsWDDk9o/P8s3MampqrKWlxRKJhPl8PkskEtba2mo1NTVDstu2bbNbb7316M/eD4dBHwAAl9H4yt6xTuMfT0VFhdXV1VljY6PNmzfPGhsbra6ubsip/T/84Q/2rW99yx544AH71Kc+JS2bC/kAAHBJJp20/43UHXfcYZs3b7Y5c+bY5s2bbfXq1WZmtnjxYnv77bfNzGz16tUWjUZt1apVNm/ePJs3b5699957KZfLTB8AgHFm5syZtmXLliG3P/TQQ0f//eSTT454uWM66CeEUyYen9Zs1dtyRMqFfFpz1L7umJQzMysLDL+OofyAdfZrrVpdYgvWBaeUSbmOXq01cNuu8PAhM6ucoLUfrl2+Tsr97ZrvSLn8PO1EVG1ZUMo994dmKfe1i4b/XMzM7EBXVMrNqiiUcm82D72y91jU/XKoQ1u/z03XjqsjPdpz5JRS7fFQL5YKiNtrZhZPaM+5Q5F+KTelRNuWiWJD3Y42rX2zNqQ17an7sEZ8Dh/p1R5jv9cj5TrF17ZQQXob/k5UNnfvc3p/hJQB38zkAR9AeqR7wAeyEaf3AQBwyebufWb6AADkCGb6AAC4KNefZSoGfQAAXD7JV+wyBaf3AQDIEcz0AQBw4St7AAAg4zHTBwDAJZu/ssegDwCAC1fvp8kpQlVlT7NWDVs6s1bKeWM9Uq46VD58yMxeORCRciNp/fr81FIpd/9LO+VlKj47s0LKffMba6Tc/f9rlZQ73K3tm//4Y4uUKw9pFaNf+ctpUq69T6sxfnnHYSlXkFct5UqDWhXpKWI1rN+rfXoXFZvsZk0sknITC/O1+41r99s7kJByfq/WlnlqWaFNCGgvfeo6qhd7XzClRMqp29zdr+XU5Z1TPUHKtYlV33WVxVLusFjxjBPHTB9ATlEHfOSuBF/ZAwAAmY63vAAAuGTzTJ9BHwAAl2we9Dm9DwBAjmCmDwCACzN9AACQ8ZjpAwDgks0zfQZ9AABcsnnQ5/Q+AAA5Ykxn+of748Nmzq7R6nALarRq02SBVit5qGf4dTMzqy7WKkZH8oMNj71+QMqdUROScs/9fq+U+7f7fyTl/nbNd6Tc/yveb7BAO+ymVWkVnlPKtFra37zfJuUumF4m5b4wq1LK+b0eKdchVpu+tr9Dys2o0GpzIzHt2N/fFZVyvQNada1P2y2W59PmJup+7oxq+9nMLJCnVfuq9ndpFdQDYjWyX9w3NcVaVfVBsSJb3NW2r1M7ZvzqwTBGmOkDAICMx2f6AAC4MNMHAAAZj5k+AAAu2TzTZ9AHAMAlmwd9Tu8DAJAjmOkDAOASZ6YPAAAyHTN9AABcsvkzfQZ9AABccn7Qb29vt0OHDpmZ2aRJk6ysTKsp/bjy/OErLfMKtJpbb3GplHMCWg2vJ649yNuP9Ei53eFeKWdm5hM7Ld/Y0y7l3nrqZ1Lu3PnXSbk9bdo2n3eqVqH8xgdaHW5CrDzujmo1sp85RTtmDoh1s/l52qdjasNogVj5quZUfq+2HZWF2uNRHEjvXEKt102Ir9OFfv1TTfW1X11Hv/AaaGaWcNRjS7xf8SAsER+7hFgzXliobe9IastxYlI+wnv37rXbb7/dmpqarKqqyszMWltb7cwzz7TVq1fb9OnTx2IdAQAYM+qbmkyUctBfvny5LVq0yB555BHz/vdsIJlM2rPPPmsrVqywxx9/fExWEgAAnLiU55A6Ojps7ty5Rwd8MzOv12vz5s2zzs7OUV85AADGWiLppP1vvEg56JeWllpjY6M5rlMdjuPYM888Y6GQ9jOvAABkkmwe9FOe3r/rrrusoaHB1qxZY9XVH/5+fUtLi51xxhl21113jckKAgCA9Eg56E+fPt02bdpk4XDYmpubzcyspqbGysu1q7QBAMg042lmnm7S9zPKy8uHDPRXX321Pfvss6OyUgAAIP1SDvrvv//+MW93HMfa27XvjAMAkEkSyeTJXoVRk3LQr6+vt9ra2kEX8n2ko6Nj1FYKAACkX8pBv7a21h577LGjF/G5XXLJJaO2UgAAnCw5+5n+lVdeaQcOHDjmoH/FFVeM+M4KCoa/hMBJaKdVnPiAlvP5pVxJICHlIv1a5euOloiUMzP7nFhf+7/W3Cfl5nxjsZT7zDStTnl7c5d2v2dUSbnSQu0xUeuJ1dxna0uk3P+3OyzlivO1ylK1hrc2VJDe+xXbZjvFGmP1fvPztA1WS88GxH5ddXuL/HqN8YD44q9uS6Ff2zdx8eyy2MIr/1TsxKC2E2Pi8orFyuOegfF1Oj1nB/0VK1Yc93+77bbb0r4yAABg9PArewAAuKhnRjKR/nNTAAAgozHTBwDAJWc/0wcAINdk86DP6X0AAHIEM30AAFyY6QMAgIzHTB8AAJdsnukz6AMA4MKgnyYt3bFhMxeddaq0LE++Vlna1qfV6247pNXmXjC1VMqNxE1fWy3l1v+oQco1HdRqc9siwz8eZmbf/OJMKff7fdqPME0u0R67GeWFUm6CWA97uEfb3r8UH+PifK3O1S/WBPeJ3atqNaxagVoe1PZfr1iVmidub4HaTyxS65hH8oJemqc9xvnitkRi2j4UGsvNzCwg3m9BnnYsdMW018vifLGuV6xQLivQq5FxYpjpAwDg4mTxTJ8L+QAAyBHM9AEAcEky0wcAAJmOmT4AAC6Ok70zfQZ9AABcuJAPAABkPGb6AAC4cCEfAADIeAz6AAC4OMn0/43Url27bOHChTZnzhxbuHCh7d69e0jm5ZdftmuuucbOOussW7dunbTcMT29P+2U0LCZngOHpWWVnTJbyhXkaTWVM8qCUu7xtw5KuR+uvFvKmZlt/N9ave7vPghLuVnVxVLu3JrhHw8zsx1tvVLu6tOrpFxb34CU83m0x24gqT2jzq3Wan0P92pVpCq1lrYkoFWRqtWrcfEUpbp+5WJVqld83HxJ7TgY8KT3Zcqf6Jezjs+vBcVtDnr7tPv1iPMx9Sx0UtuOckdbP0to61c4EJVyjkd7bpqpuRMzHq7eb2hosEWLFtm8efPs6aeftlWrVtmjjz46KDN16lT7wQ9+YL/85S8tFtNqxpnpAwAwyrq6umz//v1D/rq6hv5WSltbmzU1NVl9fb2ZmdXX11tTU5OFw4MnftOmTbO6ujrLy9PfGHMhHwAALqNxId+mTZvswQcfHHL70qVLbdmyZYNua25uturqavP5Pjy75vP5rKqqypqbm628vPyE1oNBHwCAUXbDDTfYggULhtweCmkfs6YLgz4AAC6jUc4TCoXkAb6mpsZaWloskUiYz+ezRCJhra2tVlNTc8LrwWf6AAC4OEkn7X8jUVFRYXV1ddbY2GhmZo2NjVZXV3fCp/bNGPQBABh37rjjDtu8ebPNmTPHNm/ebKtXrzYzs8WLF9vbb79tZmavvfaaXXzxxfbII4/Yz372M7v44ovtt7/9bcrlcnofAACX5Dj4yt7MmTNty5YtQ25/6KGHjv77/PPPt9/85jcjWi4zfQAAcgQzfQAAXPiVPQAAkPHGdKY/5fNTh80UT62WluWZfYGUO9IXl3JtvVolqFqv+727lks5M7MLppbKWUVxQHtYayYEpJxaq1oS0N5DejxaJWhfXKvXnVyQL+UCedr61RRr2xsZ0NavNF/cLwmtRtOT0I5VJ097fB2vdrx4BrSKVm90aMPYMe83X6tUzY9r+8V84naoyzMzR11mQnudUfdh2h+7pLh+sR7tfv3aY+eNRaRcMi5WI4dO/Op1RTbP9Dm9DwCACz+tCwAAMh4zfQAAXMbDr+yNFmb6AADkCGb6AAC4ONo1uhmJQR8AABcu5AMAABmPmT4AAC7Z/D19ZvoAAOSIlDP9rVu32oUXXmhmZt3d3bZmzRrbtm2b1dXVWUNDg02cOHFEd9bfNXwbVcG5X9AWJjY9vXRQe1+z7Ourpdyvn/wHKWdmVlGoNc990K61dNWfru3vzv6ElFO/lXJBbbGUa+nVWr+mTtAa9HxJrXnOG2mVckm/1ubl7zwo5QrENjTv4TYplywsk3Iqx6ft53Q36Hn7OrXlecSmwhFcVaUu0/K056anX2yoi2o5K66QYt5ot7Y8cd84fdryPAVF2v32dmj3m68tz9d/WLtfmynmTkzOzvTvueeeo/++7777rKioyDZu3GgzZsywtWvXjvrKZTJ1wAeQHuke8JG7ko6T9r/xIuVUxV1Q8Prrr9sTTzxhfr/fZs+ebVdfffWorxwAAEiflIN+LBaznTt3muM45vF4zO//8ztkr5fLAQAA2SebT++nHPSj0agtWbLk6Iy/paXFqqurLRKJMOgDAJBhUg76L7744jFv9/l89sADD4zKCgEAcDJl80z/E03Xg8GgTZ06Nd3rAgAARhHlPAAAuGRzDS+DPgAALvy0LgAAyHjM9AEAcMnmC/nGdNCvOu+0YTOdUz8nLSvf55Fyy+beLOXW/6hBylUVa21eE4P6ri0JaDW3JQGflCvL06o5Pf1aNae3S8sVipWbni6xXlethw03S7m8oLh+YmObWr06cOSQlPOK6+fEolLOV1Yp5RKdWk2wJ0+r9R3obpdyJi7P4rG0Ls/j1xv5kj3asa8eC96QVsNrSa1KW805ce05Z17tNSbtxO0oOO3zo7wi2Y+ZPgAALlzIBwBAjnDUMy0ZiAv5AADIEcz0AQBwYaYPAAAyHjN9AABcsnmmz6APAICLk8jeQZ/T+wAA5Ahm+gAAuGTz6X1m+gAA5IgxnemHzjln2MyAWK9bfqFWrxveukHKBdvel3JJR6u4dd58RcqZmRUPaDWjav2qiTWyiUiHlOtpbtGWN6BVfRZWT9SW198v5bp2aTW8Kn9RgZSLR8XHLaEdMwO9Yr2uX3vaquunbq9Kvd9AqVY/3deq1fo6SW0/j4SvICDlkjHt2FfXUb1f9dhKRLXnkkpdv1iXVk/sy9eO6dOuWiLlThQzfQAAkPH4TB8AAJdsnukz6AMA4JLNgz6n9wEAyBHM9AEAcGGmDwAAMh4zfQAAXJJZPNNn0AcAwIXT+wAAIOMx0wcAwCWbZ/pjOujnVUwaNhMS63XbxHrdWMKRcsHuNm15bz0t5SL7tOpaM7Oi2kopF35nl5Tz+LQTOGpdak9zWMoV1ZRLObU2N9rWLeW6myNSrqiqUMolBrRq0/4urdq090iflMsv0uqTPWJVtb9Ae3r3d4m1uaH8tC7Pmy8ep31xKafKC+ovewM9Wr2uuo4FZVrlsSO+biXVGl7xmPb5T87JXyepbe9po7weuYCZPgAALk6CmT4AADkhm0/vcyEfAAA5gpk+AAAuzPQBAEDGY6YPAIALM30AAJDxmOkDAODiJLVeg0zEoA8AgAun9wEAQMYb05l+4Y1PDpvp/ef/IS0r/vJjUq7vD3+Qcu9ue1/KBUJBKbf/9/uknJnZnr1dUq56glaDGo1qlaDhmPZutjKgHSZ7e7V96PNoNbJBsW72QJprWhOOVgla4vdJuZZ+bf1iYhWpun6TC7Ra3x61ylW8X/W4Ks/X9l8kLlbIisfVaFD3jdiua+Khb0Gxcltdv06xrrc4T7tf9bHL92obfLGUOnHM9AEAQMbjM30AAFySWTzTZ9AHAMAlm39wh9P7AADkCGb6AAC4cCEfAADIeMz0AQBwYaYPAECOcJKJtP+N1K5du2zhwoU2Z84cW7hwoe3evXtIJpFI2OrVq+3yyy+3K664wrZs2TLschn0AQAYZxoaGmzRokX2/PPP26JFi2zVqlVDMs8++6zt3bvXfvWrX9njjz9u69evt/3796dcLoM+AAAuozHT7+rqsv379w/56+oa2sja1tZmTU1NVl9fb2Zm9fX11tTUZOFweFDuueees2uvvda8Xq+Vl5fb5Zdfbr/85S9TbtuYfqYf2/Zw2palrnjBlVqu6hOvybHNSvPyAABjI51j1UfWr19vDz744JDbly5dasuWLRt0W3Nzs1VXV5vP92FVtc/ns6qqKmtubrby8vJBucmTJx/975qaGjt06FDK9eBCPgAARtkNN9xgCxYsGHJ7KBQa0/Vg0AcAYJSFQiF5gK+pqbGWlhZLJBLm8/kskUhYa2ur1dTUDMkdPHjQzjnnHDMbOvM/Fj7TBwBgHKmoqLC6ujprbGw0M7PGxkarq6sbdGrfzOyqq66yLVu2WDKZtHA4bC+88ILNmTMn5bI9jiP+5iIAABgTO3futJUrV1pXV5eFQiFbt26dzZgxwxYvXmy33HKLnX322ZZIJGzNmjW2detWMzNbvHixLVy4MOVyGfQBAMgRnN4HACBHMOgDAJAjGPQBAMgRDPoAAOQIBn0AAHIE5TwjcOmll1p+fr4FAgEzM/vud79rF1100Uleq+GtW7fOnn/+eTtw4IA9++yzNnv2bDP78FecVq5caR0dHVZaWmrr1q2z6dOnn9yVHcbxtiWTHpv29nZbvny57d271/Lz823atGm2Zs0aKy8vtzfffNNWrVpl/f39Vltba//wD/9gFRUVJ3uVjyvVtpx++uk2e/Zs83o/nFvcfffddvrpp5/kNT6+m266yfbv329er9cKCwvt9ttvt7q6uox7nhxvOzLpOYJR5ED2pS99yXnvvfdO9mqM2KuvvuocPHhwyPp/+ctfdp566inHcRznqaeecr785S+frFWUHW9bMumxaW9vd37/+98f/e+77rrL+d73vuckEgnn8ssvd1599VXHcRxnw4YNzsqVK0/WakqOty2O4zizZ892IpHIyVq1Eevq6jr673//93935s+f7zhO5j1PjrcdmfQcwejh9H4OOP/884fUN6q/4jTeHGtbMk1paaldcMEFR//705/+tB08eNDeeecdCwQCdv7555uZ2XXXXTfsL2adbMfblkw0YcKEo/+ORCLm8Xgy8nlyrO0APsLp/RH67ne/a47j2F/8xV/Yt7/97TH/sYR0UX/FKZNk4mOTTCbtX/7lX+zSSy8d0ptdXl5uyWTy6Gnl8c69LR/58pe/bIlEwi6++GJbtmyZ5efnn8Q1HN7f/d3f2datW81xHPvnf/7njH2efHw7PpKJzxGkFzP9EfjpT39qzzzzjD355JPmOI6tWbPmZK8S/lumPjZ33nmnFRYW2vXXX3+yV+WEfXxbXnrpJfvXf/1X++lPf2rvv/++bdiw4SSv4fB+8IMf2EsvvWTf+ta37O677z7Zq/OJHWs7MvU5gvRi0B+Bj04r5+fn26JFi+yNN944yWv0ybl/xcnMjvsrTpkiEx+bdevW2Z49e+wf//Efzev1Hv3FrI+Ew2Hzer0ZMcv/+LaY/fkxKS4utmuvvTYjHpOPzJ8/31555RWbNGlSRj9PPtqO9vb2jHyOIP0Y9EW9vb3W3d1tZmaO49hzzz1ndXV1J3mtPjn1V5wyQSY+Nvfee6+98847tmHDhqOnvM866yyLRqP22muvmZnZz372M7vqqqtO5mpKjrUtnZ2dFo1GzcwsHo/b888/P64fk56eHmtubj763y+++KKVlJRk3PPkeNsRCAQy7jmC0cEP7oj27dtny5Yts0QiYclk0mbOnGm33XabVVVVnexVG9batWvtV7/6lR05csTKysqstLTUfv7znx/3V5zGs2Nty49+9KOMemx27Nhh9fX1Nn36dCsoKDAzsylTptiGDRvsjTfesIaGhkFf2Zs4ceJJXuPjO9623HjjjbZq1SrzeDwWj8ftvPPOs+9///tWVFR0ktf42I4cOWI33XST9fX1mdfrtZKSEluxYoV96lOfyqjnyfG2IxQKZdRzBKOHQR8AgBzB6X0AAHIEgz4AADmCQR8AgBzBoA8AQI5g0AcAIEcw6AMAkCMY9AEAyBH/P2TOi2QivfcmAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Fi6QgHlbAih",
        "colab_type": "text"
      },
      "source": [
        "## Cosine Similarity of weights\n",
        "\n",
        "(Refer figure 2(a) from paper)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xCKbE2UNfMzg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get the weights of the input model. \n",
        "def get_model_weights(model):\n",
        "  model_weights = []\n",
        "  # iterate through model layers.\n",
        "  for layer in model.layers:\n",
        "    # grab weights of that layer\n",
        "    weights = layer.get_weights() # list\n",
        "    # check if layer got triainable weights\n",
        "    if len(weights)==0:\n",
        "      continue\n",
        "\n",
        "    # discard biases term, wrap with ndarray, flatten weights\n",
        "    model_weights.extend(np.array(weights[0]).flatten())\n",
        "\n",
        "  return np.array(model_weights)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NpIDtmE1YeLE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "06aec662cf6948189ee8e371d5287384",
            "fcf70e91da844425846d47710b3d27d2",
            "0992be684a114c469ceb05371e23c13b",
            "a0fa4a5402214313b515bf32b70a2c20",
            "edebac964ed940178ac93669431292b5",
            "4a4c4f42d6a44959b502d94692a3ef56",
            "33ddad7f5d53479788a45062f3d5a856",
            "edec4ec464d44fe188ebe8b45802e9bd"
          ]
        },
        "outputId": "b545a4ad-8065-4acf-b8af-08975d9d1a03"
      },
      "source": [
        "weights_of_models = []\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model(MODEL_PATH+same_model_ckpts[i])\n",
        "  # get predictions for model\n",
        "  weights = get_model_weights(model)\n",
        "\n",
        "  weights_of_models.append(weights)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "06aec662cf6948189ee8e371d5287384",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_9k-CujgfnEf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "9b7c1be9132046529000e7316f5dcabc",
            "9c9023cf2f554d6eaccc67a22af7e508",
            "b394ade9ec79466b9eb4e070b42796ba",
            "6a5c2a702644452284c07f0e29577e3d",
            "775fb0f3e2584cff9c818ca232559bbf",
            "b8e9cb227fbc4ba29cd35b7b3b0538f0",
            "1a72d8b8a4a8425a8be59de1ac3995d0",
            "c66e02e3967946869659f7f2e9e0002d"
          ]
        },
        "outputId": "fcf351e3-4c3a-4c7e-fe6f-01ea64485a62"
      },
      "source": [
        "empty_arr = np.zeros(shape=(40,40))\n",
        "\n",
        "for i in tqdm_notebook(range(40)):\n",
        "  weights1 = weights_of_models[i]\n",
        "  for j in range(i, 40):\n",
        "    weights2 = weights_of_models[j]\n",
        "    \n",
        "    # compute cosine similarity of weights\n",
        "    cos_sim = np.dot(weights1, weights2)/(norm(weights1)*norm(weights2))\n",
        "    \n",
        "    empty_arr[i][j] = cos_sim\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = cos_sim\n",
        "\n",
        "cos_sim_coeff = empty_arr[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9b7c1be9132046529000e7316f5dcabc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "H7GBJlnEfnE8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "d43d14bc-dc9e-4838-e2b7-8ecf91175d88"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns.heatmap(cos_sim_coeff, cmap='RdBu_r');\n",
        "plt.xticks([5,10,15,20,25,30,35],[5,10,15,20,25,30,35]);\n",
        "plt.yticks([5,10,15,20,25,30,35],[35,30,25,20,15,10,5]);\n",
        "\n",
        "plt.savefig('functional_similarity.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAHYCAYAAAAyFvBfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfXCV9Z3//9c5hyQCcghBEk8IbRSKnipuvVn9ORSdAgLjBhO2i+Okden6FYpA3G/7bSG1SACpQ1gHunKzTN0W46BdB91WExFZlulUGeu4SL2LILJYGghJJQkhCLk55/z+sKbXacLJO3gl5yTX8zFzZsLJm8/1uRJCPud9fa7X8cVisZgAAICn+ZM9AQAAkHwsCAAAAAsCAADAggAAAIgFAQAAEAsCAAAgFgQAAKSU8vJyTZ06VVdddZU+/PDDbmsikYhWrVql6dOn64477tCOHTtMn0tkiCuzBwAArpg2bZr+8R//Ud/61rcuWFNZWaljx45p9+7dampqUlFRkW699Vbl5eUl/FwidAgAAOhjzc3Nqqmp6fJobm7uUnvTTTcpFAolHG/nzp2aO3eu/H6/srKyNH36dO3atavHzyXSrx2C9Ovv67Fm8rzvmMZ6YOoEU92UL4001V3WfspU5ztx0FTXdnC/qU6SGg9+bKo7feSEqe5c43lT3XljnXW81uZW23htEVNdS0fUNl7EFrZpHa8tahuv3RjyGTHXmcpSfjwr63Hd5vZ5oP9sjX3cL8dZ6Mt3fcxrHv9/2rRpU5fnlyxZopKSkl6PV1tbq9zc3M4/h0IhnTx5ssfPJcIlAwAA+ti8efM0Z86cLs8Hg8EkzKZ7LAgAAOhjwWDQ1V/+oVBIJ06c0HXXXScpviuQ6HOJsIcAAACHgM/9h9tmzZqlHTt2KBqNqqGhQXv27NHMmTN7/FwidAgAAHAI+PrgN3gvrFmzRrt379Ynn3yif/qnf1JmZqZeeuklzZ8/Xw8++KAmTZqkwsJCvf3225oxY4YkafHixRo3bpwkJfxcIr7+fPtjNhV2j02F3WNT4cAcz4pNheit/tpU+GDgCtfHfDxy1PUx3UaHAAAAh75o8Q8E7CEAAAB0CAAAcEr2HoJkYUEAAIADlwwAAIBnDdgOgXUFZ+78xGw70BW11cWitp30vRGNGI9trDOPZzxnK/uudnfrrNyfn9vHtY3nNXxd4BavXjKgQwAAAAZuhwAAgL7AHgIAAOBZdAgAAHDw6h4CFgQAADh4tXXu1fMGAAAOdAgAAHDw6iUDOgQAAIAOAQAATl697ZAFAQAADl69ZNCvCwKfP9Bjjd9v+0b43f6GWeNvjZHEvYn7dTsaOBq1nUvMWGcVM2bHuh0x63Y08GCRrPP12tcZGCzoEAAA4ODVSwZsKgQAAHQIAABwYg8BAADgkgEAAPAuOgQAADh49ZIBHQIAAECHAAAAJ6/uIWBBAACAg1cXBFwyAAAAqRdd7DNGFwesEcemKkkxW3xwzBhd3BuxiPXYxkhiY5Zv1O06lyNr3Y44trIGSbsdmZys803Wca1SfX4YfNhUCAAAPIs9BAAAOLCHAAAAeBYdAgAAHLy6h4AFAQAADlwyAAAAnkWHAAAAB69eMqBDAAAA6BAAAODk1T0EKbcgMCcQGls6PmudOanQVpcx4TqdP/R725jGpEJ7sqDxXMx11qQ9t+tMZSmfBOg26/kOluMC/Y1LBnCVdTEAAEAqSLkOAQAAyWTtQA82dAgAAAAdAgAAnHwe3VXIggAAAAe/RxcEXDIAAAB0CAAAcPIFvPla2ZtnDQAA4tAhAADAwaubCukQAACAxB2Cffv2afLkyZKkM2fOaPXq1Tpw4IDC4bDKysp02WWX9epgPn+gxxprdLF1AWccTjJHF0dsdcZYYEmKGuOQezOmm9w+bqpHCLsdwZwsqf91TvYMgO6lwl0GR48eVWlpqZqampSZmany8nLl5+fH1fzpT3/SihUrVFNTo46ODi1cuFCFhYWSpI0bN+qZZ55Rdna2JOmGG25QWVlZwmMm7BA89thjnR9v2LBBw4cP15YtW3TllVdqzZo1F3OOAACkNJ/f7/qjt8rKylRcXKxXXnlFxcXFWrFiRZeatWvX6tprr1VlZaWefvppbdiwQbW1tZ2fLyoq0gsvvKAXXnihx8WA1MOCIOZ4BbR//379+Mc/1sSJE/W9731PR44c6c25AQDgWc3NzaqpqenyaG5u7lJ76tQpVVdXq6CgQJJUUFCg6upqNTQ0xNUdPHhQU6ZMkSRlZWXp6quv1ssvv3zRc0x4yaCtrU1HjhxRLBaTz+dTWlpa5+f8F7HiAQAg1fXFJYOKigpt2rSpy/NLlixRSUlJ3HO1tbXKyclRIPDZZfZAIKDs7GzV1tYqKyurs+6aa67Rzp07NWnSJNXU1OjAgQPKy8vr/PxLL72k1157TWPGjFFJSYmuv/76hHNMuCA4f/685s+f3/nnuro65eTkqKWlhQUBAABG8+bN05w5c7o8HwwGL3rM0tJSPfrooyosLFRubq5uvfXWzkXEPffco4ULFyotLU379u3TokWLtHPnTo0aNeqC4yVcEOzdu7fLc6dPn1ZGRoYef/zxiz4JAABSVV/cdhgMBs2//EOhkOrq6hSJRBQIBBSJRFRfX69QKBRXl5WVFbfXb/78+ZowYYIkacyYMZ3PT548WaFQSIcPH9bNN998weMmfJl/8OBB/f3f/73mzp2rI0eOaMGCBbrttts0c+ZMnT171nRiAAAMJL6A3/VHb4wePVrhcFhVVVWSpKqqKoXD4bjLBZLU2Niojo4OSdLrr7+uDz/8sHPfQV1dXWfdBx98oOPHj+uKK65IeNyEHYI1a9Zo8eLFOnPmjO6//35973vf089+9jPt3btXa9eu1ZNPPtmrkwQAAD1buXKlSktLtWXLFgWDQZWXl0v6rAvw4IMPatKkSXrnnXf0k5/8RH6/X6NGjdLWrVs1dOhQSdL69ev1/vvvy+/3Ky0tTevWrYvrGnQn4YLg7NmzmjZtmiTpX//1X3XXXXdJkqZOncolAwDAoJQKOQTjx4/Xjh07ujz/xBNPdH58++236/bbb+/273++gOgN822HnwcUfc4apgMAAFJfwg7B2LFj1dLSoksvvTQuiOjkyZOdbQkAAAYTnznidnBJuCDYvHlzt88Hg0Ft2bKl1wdzM7rY77PVmb+txuhiWaOLe9FBsUYDx4xZr27XWbkdRWuPEHZ3PLel/vzIEAac/B59++OLerfDYcOGadiwYW7PBQAAJAlvfwwAgANvfwwAADyLDgEAAA50CAAAgGfRIQAAwIG7DAAAAJcMAACAd9EhAADAwe/RpEI6BAAAYOB2COwRx8YBzdHF1pjh3kQX2+KQoy5HElvHs9bZo4bdjkxO7QhmtzE/oG/52FQIAABS4e2Pk8GbyyAAABCHDgEAAA7cdggAADyLDgEAAA5sKgQAAGwqBAAA3kWHAAAABx9JhQAAwKvoEAAA4MDbH/cDXyDQY401kth8TJ9tPJ8xutgaM9yr6GKX45Cjxjr7ca2RxKYy1+us7N8RG/t5JCfLN1nHBTAw0SEAAMDBq8FELAgAAHDwag6BN88aAADEoUMAAICDz+/N18rePGsAABCHDgEAAA7cdggAANhUCAAAvIsOAQAADnQIAACAZ/Vrh8Dvdy+62JpwbE5CNsb4KmqLLo4aI46lXkQSR21RtDFjXdSYvRtN8ehdtyOOkyXVzyPV5we4xau3HXLJAAAAB8v77gxG3lwGAQCAOHQIAABwYFMhAADwLDoEAAA4+D26qdCbZw0AAOLQIQAAwMGrewhYEAAA4ODVBYE3zxoAAMQxdQgaGxt18uRJSdLll1+uUaNG9emkAABIFpIKu3Hs2DE9/PDDqq6uVnZ2tiSpvr5eX/3qV7Vq1Srl5+f36mA+Q3TxEGPWcMBnq7MmF8saz2uMLo5Zo5Bljy6OWaOGjXXW8ewRwqkdNez2eQDAYJJwQbB06VIVFxdr27ZtnbdhRKNRVVZWatmyZXr22Wf7ZZIAAPSXVNhDcPToUZWWlqqpqUmZmZkqLy/v8iL8T3/6k1asWKGamhp1dHRo4cKFKiwslCRFIhGtWbNGr776qnw+nxYsWKC5c+cmPGbCs25qatJdd90Vd0+m3+9XYWGhTp8+fZGnCQBA6vIF/K4/equsrEzFxcV65ZVXVFxcrBUrVnSpWbt2ra699lpVVlbq6aef1oYNG1RbWytJqqys1LFjx7R79249++yz2rhxo2pqahIeM+EsMzMzVVVVpZijhRqLxfTiiy8qGAz2+gQBAPCi5uZm1dTUdHk0Nzd3qT116pSqq6tVUFAgSSooKFB1dbUaGhri6g4ePKgpU6ZIkrKysnT11Vfr5ZdfliTt3LlTc+fOld/vV1ZWlqZPn65du3YlnGPCSwZr165VWVmZVq9erZycHMViMdXV1SkcDmvt2rX2rwQAAAOEvw8uGVRUVGjTpk1dnl+yZIlKSkrinqutrVVOTo4Cf37XxUAgoOzsbNXW1iorK6uz7pprrtHOnTs1adIk1dTU6MCBA8rLy+scIzc3t7M2FAp13hxwIQkXBPn5+aqoqFBDQ0NnGyIUCsVNCAAAJDZv3jzNmTOny/NfpNteWlqqRx99VIWFhcrNzdWtt97auYi4GAkXBAcPHtRDDz0kv9+v8vJylZeX64033lBmZqa2bt2qcDh80QcGACAV9cVth8Fg0PzLPxQKqa6uTpFIRIFAQJFIRPX19QqFQnF1WVlZeuyxxzr/PH/+fE2YMKFzjBMnTui6666T1LVj0J2EZ71mzRotXrxY3/72t3X//feroKBAb7/9tsrKylReXm46MQAABpJkbyocPXq0wuGwqqqqJElVVVUKh8NduvONjY3q6OiQJL3++uv68MMPO/cdzJo1Szt27FA0GlVDQ4P27NmjmTNnJjxuwlmePXtW06ZNU1FRkSTprrvukiRNnTpVTU1NvTpBAABgs3LlSm3fvl0zZ87U9u3btWrVKkmfdQHeffddSdI777yjO++8U7NmzdLjjz+urVu3aujQoZKkwsJC5eXlacaMGbr77ru1ePFijRs3LuExE14ycN5dMHny5LjPRXsRvAMAwECRCjkE48eP144dO7o8/8QTT3R+fPvtt+v222/v9u8HAoHORYRVwrMeO3asWlpaJH12+eBzJ0+e7FyFAACAgS9hh2Dz5s3dPh8MBrVly5Y+mVDAGF3sN0YXW+sUM8YHGzsj0bYO23ElRY3RxdY6axSy26yRxG5HHCcrajhZUc1un2+yoqSBzwXMGfP9g/cy6IVhw4Zp2LBhbs8FAAAkyUUtCAAAGKz8X+Be/oGMBQEAAA6psKkwGbx51gAAIA4dAgAAHOgQAAAAz6JDAACAA7cdAgAALhkAAADv6tcOgX9IWo816UNs93+mGaOt/LLFsPmixmTBjjZTmTXRUJKi7bZjx4yRcta6SFvEVNcWtSbyJSeB0PqVth/XOKDLSCDEQJFqyYJuo0MAAAA8iz0EAAA4sKkQAADI5/dmdLE3l0EAACAOHQIAAJzoEAAAAK+iQwAAgJNHNxV686wBAEAcOgQAADj4At7cQ8CCAAAAJ49uKuzn6OL0HmvSh9iuYqQZr/GYI4kj7aayWIetLtJmPK7sEcLmunZbnTni2OXI32RFHLtfZyojkhhJN9ijhuEOOgQAADh5tEPApkIAAECHAAAAJ97LAAAAcMkAAAB4Fx0CAACc6BAAAACvokMAAIADmwoBAACXDAAAgHf1a4fAZ1h1Bfy2jM00axZn1Bjja4wkjrW32Q7b3ovo4vaou3Vttrr2qK2uLWrLyrXWJS/i2FZnRSQx+spgiRoO+AboidAhAAAAXsUeAgAAHLz69sd0CAAAAB0CAADicNshAABgUyEAAPAsOgQAADhYbpEfjOgQAAAAOgQAAMRhUyEAAPDqJYOEC4LDhw/rK1/5iiSpvb1dW7Zs0VtvvaVwOKx//ud/1tChQ3t1MH9aeo816QHbyswaiemLuhtJLGPEcbTNHl0cbbPFK1vrYhFjxHHSIoRthbaz6M383D0P4HNEDWMwSPjbd+nSpZ0fb9y4UYcOHdJ3vvMdNTU16dFHH+3zyQEA0O/8AfcfA0DCDkHM8Yrq1Vdf1fbt2zV8+HBNmTJFRUVFfT45AADQP3pcEJw/f16xWEyBQEDDhw//7C8NGaIhQ9h+AAAYhFJgU+HRo0dVWlqqpqYmZWZmqry8XPn5+XE1p06d0o9+9CPV1taqo6NDt9xyi5YvX64hQ4Zo48aNeuaZZ5SdnS1JuuGGG1RWVpbwmAl/qx86dEjXX3+9YrGYfD6f6urqlJOTo9bWVkWNb50LAMBAkgpvblRWVqbi4mIVFhbqhRde0IoVK/TUU0/F1WzdulXjx4/Xz372M7W3t6u4uFi7d+/WnXfeKUkqKirSsmXLzMdMuCA4ePBgt8+3trZq9erV5oMAAOBlzc3Nam5u7vJ8MBhUMBiMe+7UqVOqrq7Wtm3bJEkFBQV65JFH1NDQoKysrM46n8+ns2fPKhqNqq2tTe3t7crJybnoOSbsizQ2Nmr58uW677779PTTT8edwM9//vOLPigAACmrDzYVVlRUaNq0aV0eFRUVXQ5fW1urnJwcBf7cqQgEAsrOzlZtbW1c3aJFi3T06FF9/etf73zceOONnZ9/6aWXNHv2bN133306cOBAj6edsENQVlamvLw83XbbbfrlL3+p119/XT/96U81ZMgQ1dTUmL6uAAB43bx58zRnzpwuz/91d6A3du3apauuukoVFRU6e/as5s+fr127dmnWrFm65557tHDhQqWlpWnfvn1atGiRdu7cqVGjRl1wvIQdgo8//lhLly7VjBkz9Itf/EJjxozRd7/7XbW2tsbdgQAAwKDRBx2CYDCovLy8Lo/uFgShUEh1dXWKRD7LnolEIqqvr1coFIqr2759u+666y75/X6NGDFCU6dO1RtvvCFJGjNmjNLS0iRJkydPVigU0uHDhxOfdqJPtrf/JYTH5/OprKxMEydO1IIFC9Ta2mr4qgIAMLD4/H7XH70xevRohcNhVVVVSZKqqqoUDofj9g9IUl5enn77299Kktra2vT66693hgnW1dV11n3wwQc6fvy4rrjiioTHTTjLcePG6c0334x7btmyZfqbv/kbffzxx7YzAwAAvbJy5Upt375dM2fO1Pbt27Vq1SpJ0vz58/Xuu+9Kkh566CHt379fs2fPVlFRkfLz83X33XdLktavX6+CggLdddddWr58udatW6cxY8YkPKYvlqD339TUJJ/Pp5EjR3b53EcffaQJEyb06gSz56zvseY79800jXXfzeNMdVemtZjq/Ed73nAhSefef7PnIkmfvHPEVCdJDYdOmuoa/7fJVHe+8byprrndFoXc0mG7xdTtunbjZam2aHLqrIhCTi2DJWZYSl7UcLK+hhs6jvbLcaIf/c71Mf0T/j/Xx3Rbwk2FmZmZF/xcbxcDAAAgdRE3CACAky/5SYXJ4M2zBgAAcegQAADg5NEOAQsCAAAcYh5dEHjzrAEAQBw6BAAAONEhAAAAXkWHAAAApyQFPiUbCwIAAJx6+d4Dg0W/LggCQ9J7rEkfYvtGDPHbVnC+SIepLtbR3nORpGi7bbxou208SYoYI4Stde1RWzSw21G+1ohe2+zs47ldZ0UkcWoZCJHEXosaTtb54uLQIQAAwIHbDgEAgGfRIQAAwMmjHQIWBAAAOHl0QeDNswYAAHHoEAAA4ESHAAAAeBUdAgAAHLjtEAAAeBYdAgAAnDzaIejXBYHfxehicxRnxBb3G2s7b6qLttmiizvOtZnqJCnSZptj1Fhnj/J1N5LYHnGcnDoMTG7H7g6EOF2vRQ2nXOz0APg30he8uQwCAABxuGQAAICTRy8ZePOsAQBAHDoEAAA4ePW2QxYEAAA4+b25IPDmWQMAgDh0CAAAcPLoJQNvnjUAAIhDhwAAACePdgh8sVj/xbyNu/dJU93i+yab6or/JmSqy22tNdVFPnjdVHfmvbdNdX86cNhU13C40VTX+L9NprrmdluiYUtHNCl1542JhtbkQ/cTEk1lcMFgSSEkWfCLjmcb8NHWI+4e+ALaTn/i+pjpIy9zfUy3pdwyyGuLAcCrBstiABgsuGQAAICDV3MIvHnWAAAgDh0CAACc6BAAAACvokMAAICTRzeosiAAAMCJSwYAAMCr6BAAAODAbYcAAMCzzB2C5uZmSVIwGLzog/kCgR5r0ofY1ijmVLJIm6ks1tFuqou2dRgPa4sPlqSIMWo4ao7eTU40sC24uDcRwmQIp5LBFM87WM6F8+gjHu0QJFwQNDQ06LHHHtPLL78sSYrFYvL7/Zo1a5Z+8IMfKCsrq18mCQBuSblfPkg5MY/eZZBwGfTDH/5Q48aN0969e3XgwAH9/ve/1549e5SXl6cf/vCH/TVHAADQxxIuCI4fP64HHnhAo0aN6nwuKytLixYtUk1NTZ9PDgCA/haLuf8YCBIuCDIyMnTgwIEuz7/11ltKT0/vs0kBAID+lXAPwapVq7R06VJlZGRo7Nixkj7rGrS2tqq8vLxfJggAQH+ybuAebBIuCL72ta/plVde0Xvvvafa2lpJUigU0rXXXiufRzddAAAGt1RYDhw9elSlpaVqampSZmamysvLlZ+fH1dz6tQp/ehHP1Jtba06Ojp0yy23aPny5RoyZIgikYjWrFmjV199VT6fTwsWLNDcuXMTHjPhJYPGxkY9/PDD2rBhg+rr6zVjxgxNmjRJPp9PJSUlX/iEAQBAV2VlZSouLtYrr7yi4uJirVixokvN1q1bNX78eFVWVurFF1/U+++/r927d0uSKisrdezYMe3evVvPPvusNm7c2OPev4QLgrKyMgWDQd1zzz367//+by1ZskQdHZ/dh//HP/7xYs8TAICUFY25/2hublZNTU2Xx+cZP06nTp1SdXW1CgoKJEkFBQWqrq5WQ0NDXJ3P59PZs2cVjUbV1tam9vZ25eTkSJJ27typuXPnyu/3KysrS9OnT9euXbsSnnfCBcHHH3+spUuXasaMGfrFL36hMWPG6Lvf/a5aW1t79cUFAMDLKioqNG3atC6PioqKLrW1tbXKyclR4M9hfoFAQNnZ2Z2X7j+3aNEiHT16VF//+tc7HzfeeGPnGLm5uZ21oVBIJ0+eTDjHhHsI2tv/kt7n8/lUVlam8vJyLViwgEUBAGBQivXBpsJ58+Zpzpw5XZ7/Ium/u3bt0lVXXaWKigqdPXtW8+fP165duzRr1qyLGi/hgmDcuHF688039bd/+7edzy1btkzr16/XE0880euD+Yf0fKvi0LSe440lKc2YLOmL2KKGY23nTXUd521RyJF2a5CvPebY7ajhiPHfvL0uOVtx7FHIfTyRASrV42/dnl9fRCHbj52s46Z21HAyvyf9JRgMmn/5h0Ih1dXVKRKJKBAIKBKJqL6+XqFQKK5u+/btevTRR+X3+zVixAhNnTpVb7zxhmbNmqVQKKQTJ07ouuuuk9S1Y9CdhL9W161bp4kTJ3Z5/vvf/74qKytNJwYAwEDSF3sIemP06NEKh8OqqqqSJFVVVSkcDnd5u4C8vDz99re/lSS1tbXp9ddf11e+8hVJ0qxZs7Rjxw5Fo1E1NDRoz549mjlzZsLjJlwQZGZmauTIkd1+bsKECbYzAwBgAIn1waO3Vq5cqe3bt2vmzJnavn27Vq1aJUmaP3++3n33XUnSQw89pP3792v27NkqKipSfn6+7r77bklSYWGh8vLyNGPGDN19991avHixxo0bl/CY5nc7BAAA/WP8+PHasWNHl+edl+u/9KUvadu2bd3+/UAg0LmIsGJBAACAQ29b/IOFN9/0GQAAxKFDAACAQ1/cdjgQsCAAAMDBftP44MIlAwAAQIcAAAAnj14xoEMAAAD6uUOQltFzdPEwY3RxujE709d+zlQXO3fWVNdx1hZx3N5iiziWpLaz7T0XSToXsV3ZOmfM6LWO125cLrsdrex2BPNgkerxt8mKte2Lr0uyInVTPRrY7VeSqRZd7NXbDrlkAACAg1fvMuCSAQAAoEMAAIATtx0CAADPokMAAICDR7cQ0CEAAAB0CAAAiBP1aIuABQEAAA7eXA5wyQAAAIgOAQAAcUgq7AfpGT0f7tJ0W3RxxhBbcyN6ttlU19HSYqprO2OLOLbGEUtSe4s1utgaSUzUcCpJ9RjaVI8aHkzRxake+Zvq/1bRt+gQAADg4NE9hSwIAABwinp0WyGbCgEAAB0CAACcvHrJgA4BAACgQwAAgBO3HQIAAC4ZAAAA76JDAACAA7cdAgAAz+rf6OKhluhi25T8bZ+a6qKfnjHVmSOJm8+Z6lqbW011knS2I2Kqa+mImurORWx155MUSWwdL9W5Hb2brOMOluhi66ubvojTTfWvodvHtY+XnON+UewhAAAAnsUeAgAAHKIebRGwIAAAwMF41XXQ4ZIBAACgQwAAgJNXLxnQIQAAAHQIAABwini0Q8CCAAAABy4ZAAAAz6JDAACAg1dvO+zXBUHG0LQea0ZkBExj+azRxWeaTHWtjS22OmMkcdvZdlOdJJ0zZv5a69qN7S5rhLA9uji122xux7ym+92OeU1O5G+qRwinelRzMo+dvKhhVw+bctHFXpXwZ3ffvn2dH585c0Y//OEPNX36dJWUlOiTTz7p88kBANDforGY64+BIOGC4LHHHuv8eMOGDRo+fLi2bNmiK6+8UmvWrOnzyQEA0N8isZjrj4Eg4SWDmOMk9u/fr+eee05paWmaOHGiZs+e3eeTAwAA/SPhgqCtrU1HjhxRLBaTz+dTWtpf9gD4/dygAAAYfAbJO7T3WsIFwfnz57VgwYLOTkFdXZ1ycnLU0tLCggAAgEEk4YJg79693T4fCAS0cePGPpkQAADJFPFoi+CibjscOnSoHnjgAVVWVro9HwAAPO/o0aMqLS1VU1OTMjMzVV5ervz8/LiapUuX6tChQ51/PnTokDZv3qxp06Zp48aNeuaZZ5SdnS1JuuGGG1RWVpbwmAkXBB999NEFP9fY2NjT+QAAMOCkwm2CZWVlKi4uVitL8EUAABgCSURBVGFhoV544QWtWLFCTz31VFzNunXrOj8+ePCg5s2bpylTpnQ+V1RUpGXLlpmPmXBBUFBQoLFjx8bdbfC5piZb4A8AAAOJMQOuV5qbm9Xc3Nzl+WAwqGAwGPfcqVOnVF1drW3btkn67HfxI488ooaGBmVlZXU7/nPPPafZs2crPT39oueYcEEwduxYPfPMM8rJyenyudtvv/2iDwoAgJdUVFRo06ZNXZ5fsmSJSkpK4p6rra1VTk6OAoHPknsDgYCys7NVW1vb7YKgra1NlZWVevLJJ+Oef+mll/Taa69pzJgxKikp0fXXX59wjgkXBDNmzNDx48e7XRDccccdCQfuTsbQnrcsjMiwbWvwt5011UU+7boi607bGVsUsjWSuL3FHl3c0mELzj4ftdUlK5K4L1bVbnI7XnawRBdb7xeynm+qRxL3JibX/Yhea12yIokHx3G/qL64ZDBv3jzNmTOny/N/3R24GHv27FFubq7C4XDnc/fcc48WLlyotLQ07du3T4sWLdLOnTs1atSoC46T8LdvomsPy5cvv4hpAwDgPd1dGriQUCikuro6RSIRBQIBRSIR1dfXKxQKdVv//PPP65vf/Gbcc2PGjOn8ePLkyQqFQjp8+LBuvvnmCx6XMAEAABwi0Zjrj94YPXq0wuGwqqqqJElVVVUKh8PdXi44efKk9u/f3yU9uK6urvPjDz74QMePH9cVV1yR8Li8/TEAAA6pcJfBypUrVVpaqi1btigYDKq8vFySNH/+fD344IOaNGmSJOlXv/qVvvGNb2jkyJFxf3/9+vV6//335ff7lZaWpnXr1sV1DbrDggAAgBQzfvx47dixo8vzTzzxRNyfH3jggW7//ucLiN5gQQAAgEOqb5DuK+whAAAAdAgAAHBKhT0EycCCAAAAh6hH39yISwYAAKB/OwSXDE3rsWboENsaxddiSyrsaD5tqmttOmOqG/nlkap/t77HuuC4Ear5X9v7PZwz7mDxWgJhspLsrIl8bif3WVfn1vHs83P3uG5/nXuTFpjMVMNkHNf9JEXjv1Vjnc/tCfYTNhXCxLIYkGReDABwR7J+iQKDBXsIAABw8OqmQjoEAACADgEAAE7W/VWDDQsCAAAcuO0QAAB4Fh0CAAAcuO0QAAB4Fh0CAAAcvHrbIQsCAAAcuMugH2QOcy+6WOeaTWXWSOLW5lZX61o6oqY6STobsdW6H11sKnOd2/GtbkcNW+vSkhblm9oRwsmKVraO1xvJivL1uxyn6AsYI+GTNT8/V69TAR0CAAAcItx2CAAAvIoOAQAADl7tELAgAADAwasLAi4ZAAAAOgQAADjRIQAAAJ5FhwAAAAevdghYEAAA4ODVBQGXDAAAQP92CLIuzeixZugQWyRm5PQpU935U7aI4/ON5011rafbTHW9iS4mkrh7bkcNX+LyeEONcbCDJULY7fmlGeNqrXG6gfSA7cC94HaUrzWi1zxekiKO7edr/fql1mtTOgQAAMCz2EMAAIADHQIAAOBZdAgAAHDwaoeABQEAAA5eXRBwyQAAANAhAADAqYMOAQAA8Co6BAAAOHh1DwELAgAAHFgQJNDY2KiTJ09Kki6//HKNGjXqog42enh6jzXDjNHF0TNNprrWpjO2uuZWU11ze8RUd64X+cHnIraY41SPJHY7Atdal2aM6LVGDQ81nrC1zh75m9oRwoF0a9Swu+MF0myRxNYYX+v8pORF9NrPxd3oYrfn5zd+XXrzPUHfSbggOHbsmB5++GFVV1crOztbklRfX6+vfvWrWrVqlfLz8/tjjgAA9JtIjA5BF0uXLlVxcbG2bdsm/59fRUSjUVVWVmrZsmV69tln+2WSAACgbyXs0zQ1Nemuu+7qXAxIkt/vV2FhoU6fPt3nkwMAoL9FojHXHwNBwgVBZmamqqqqFHO0T2KxmF588UUFg8E+nxwAAP3NqwuChJcM1q5dq7KyMq1evVo5OTmSpLq6Ol199dVau3Ztv0wQAAD0vYQLgvz8fFVUVKihoUG1tbWSpFAopKysrH6ZHAAA/W2gvKJ3m+m2w6ysrC6LgNmzZ6uysrJPJgUAAPpXwgXBRx991O3zsVhMjY2NfTIhAACSKRK1ZcMMNgkXBAUFBRo7dmzcpsLPNTXZgoEAAEDvHD16VKWlpWpqalJmZqbKy8u7ZP8sXbpUhw4d6vzzoUOHtHnzZk2bNk2RSERr1qzRq6++Kp/PpwULFmju3LkJj5lwQTB27Fg988wznRsKnW6//fZenBoAAANDKuwhKCsrU3FxsQoLC/XCCy9oxYoVeuqpp+Jq1q1b1/nxwYMHNW/ePE2ZMkWSVFlZqWPHjmn37t1qampSUVGRbr31VuXl5V3wmAkXBDNmzNDx48e7XRDccccdvTo5Scq6tOfoYn+rLWq44/QpU11r41lT3bnG86a60+22VtJZYxyxJLUl6R9fqkcSX2KM1LVGCF86xO3oYtt41vP1G6OLrZG//nRb5K81Gtga42s/rjXi2N3oYms8b6/GdDmi1xyZ7HJcs9tRzVa9+Z70h2QvCE6dOqXq6mpt27ZN0mfd+kceeUQNDQ0X3NT/3HPPafbs2UpP/+z37M6dOzV37lz5/X5lZWVp+vTp2rVrl+6///4LHjfhgmDZsmUX/Nzy5ct7PCkAACA1Nzerubm5y/PBYLBLrk9tba1ycnIUCHy2GA4EAsrOzlZtbW23C4K2tjZVVlbqySefjBsjNze388+hUKjzPYkuhHc7BADAoaMPOgQVFRXatGlTl+eXLFmikpKSLzT2nj17lJubq3A4/IXGYUEAAEAfmzdvnubMmdPl+e5Sf0OhkOrq6hSJRBQIBBSJRFRfX69QKNTt2M8//7y++c1vdhnjxIkTuu666yR17Rh0J7Uu3AAAkGR9EV0cDAaVl5fX5dHdgmD06NEKh8OqqqqSJFVVVSkcDnd7ueDkyZPav3+/Zs+eHff8rFmztGPHDkWjUTU0NGjPnj2aOXNmwvOmQwAAgEOyNxVK0sqVK1VaWqotW7YoGAyqvLxckjR//nw9+OCDmjRpkiTpV7/6lb7xjW9o5MiRcX+/sLBQb7/9tmbMmCFJWrx4scaNG5fwmCwIAABIMePHj9eOHTu6PP/EE0/E/fmBBx7o9u8HAgGtWrWqV8dkQQAAgEMqdAiSgT0EAACADgEAAE5e7RCwIAAAwIEFQT+YMHp4jzWBphOmsU599Adb3eEGU119wzlT3Z9aO0x1LR326OKI8d+e21HDbkfvDjeOl6yo4eFDbBG4aZemmerSh9vqrNG75ghhY3Sx68dN8ehif5rtv7PeRRfbas11xjhuX8Aa12yMJDYfNznni9RAhwAAAIeYRzsELN8AAAAdAgAAnKJ0CAAAgFfRIQAAwCEW82aHgAUBAAAObCoEAACeRYcAAAAHNhUCAADPokMAAIBDzB40O6j064LgilFDe6yJfHTQNFbjB8dsdR+fNtUd+9QWSdzYHjHVtfWi5WSNBrZG+SYrQthaFzRG5bodIZwRzHB1vLRL0011yYroHTLUNj9/mjGC2RoNnG6sM44XMI5njcm1zq9XY7ocNex+FLK7dVbm+fltX7/+4tW7DLhkAAAAuGQAAIATmwoBAIBn0SEAAMDBq8FELAgAAHDw6oKASwYAAIAOAQAATlFuOwQAAF5FhwAAAAf2EAAAAM/q1w5BriGK9tyH75nGqq/+xFT3UUubqe6TNlt08bmIbeVojQWWpFHGKN8xGbZv10hjVK41QjhjpC0C1xoN7HaEsHm8YM/R2ZKUPmK4qW7I8EtsdZcYI4SNkboBY9Sw6xHCGbavs4bY5udLs31dfMbxNMQ4njFOV5JkjdQ11vmMEceyztF6XPN5uBw17PJx+4tXOwRcMgAAwIGkQgAA4Fl0CAAAcODdDgEAgGfRIQAAwCEWTfYMkoMFAQAADmwqBAAAnkWHAAAAB6/mENAhAAAAiTsE+/bt0+TJkyVJZ86c0erVq3XgwAGFw2GVlZXpsssu69XBRrQ29Fjzx3c+Mo114g+nbXXnk5dAeJkxKe5Lw2x12Vm2pL2ho2wJetaEv0tcHi8jc4SpLn3EMNt4oy41jmdMILzUNp5vqG08X7rt62dN5DMn/Ll83KQlEAaMdZJifmPT02d8LeQz/rwbk/Zi5uMmp846P/t5GL9+1vH6CR2Cbjz22GOdH2/YsEHDhw/Xli1bdOWVV2rNmjV9PrmBzLoYAOAO1xcD8KxoLOb6YyBI+BPkDGfYv3+/nnvuOaWlpWnixImaPXt2n08OAAD0j4QLgra2Nh05ckSxWEw+n09pjjdV8afYm1EAAOAGr14ySLggOH/+vBYsWNDZKairq1NOTo5aWlpYEAAAMIgkXBDs3bu32+cDgYAef/zxPpkQAADJ5NUOwUW9zB86dKjGjRvn9lwAAECSsBUeAAAHr0YXsyAAAMCBtz8GAACeRYcAAAAHr24q7N8FweE3eiw59tujpqE+ONNqqmvpsL2x9aVDbM2S3EtsX7IJl9riWyVpVP5IU93or2SZ6jJG2SJ1rRHCl4wOujrekKDtfH3DbMf1j8i01Q2zzc8/3HbcWJotSjoWMP6YBWz/ZsyJfP6AqSya4vOzjheVLSa3N6lx1krr7w9rK9o8nq3M/fkZj2tl/ZaEXD4u4tEhAADAIRU2FR49elSlpaVqampSZmamysvLlZ+f36Vu586d+rd/+7fOAMFt27bpsssu08aNG/XMM88oOztbknTDDTeorKws4TFZEAAA4BCLRpI9BZWVlam4uFiFhYV64YUXtGLFCj311FNxNe+++642bdqkiooKjRkzRmfOnFF6+l86eUVFRVq2bJn5mGwqBACgjzU3N6umpqbLo7m5uUvtqVOnVF1drYKCAklSQUGBqqur1dAQ/47BTz75pO677z6NGTNGkjRixAhlZNjedbY7dAgAAHDoiw5BRUWFNm3a1OX5JUuWqKSkJO652tpa5eTkKBD4bJ9NIBBQdna2amtrlZX1l71kR44cUV5enr71rW/p008/1R133KEHHnhAvj+/7fRLL72k1157TWPGjFFJSYmuv/76hHNkQQAAQB+bN2+e5syZ0+X5YNC2ibk7kUhEhw4d0rZt29TW1qb7779fubm5Kioq0j333KOFCxcqLS1N+/bt06JFi7Rz506NGjXqguOxIAAAwKEvOgTBYND8yz8UCqmurk6RSESBQECRSET19fUKheLvs8jNzdWsWbOUnp6u9PR0TZs2Te+8846Kioo6LyNI0uTJkxUKhXT48GHdfPPNFzwuewgAAHCIRSKuP3pj9OjRCofDqqqqkiRVVVUpHA7HXS6QPttb8NprrykWi6m9vV2/+93vdPXVV0v67N2JP/fBBx/o+PHjuuKKKxIelw4BAAApZuXKlSotLdWWLVsUDAZVXl4uSZo/f74efPBBTZo0SX/3d3+n9957T3feeaf8fr++/vWv6x/+4R8kSevXr9f7778vv9+vtLQ0rVu3Lq5r0B1frB9Dm9tef77Hmn2L1pnG2vvBJ6a6P7XaVmYEE10YwUTdGyzBRCk/P4KJLjyerWzwBBNl2v5v+6LyvvVz18esefr/uD6m27hkAAAA+veSQf2uXT3W/M/hRtNY1lf+YzJsr0LCI2z3buZ+2fbqNvurl5nqJGlU+EumupETvmyq848cbaszvrIOGMfTUOMr6wzbKj+abquLpQ8z1bUb61qNcddtEdvLmnbjy6mI8WVSxPhyzzg9dbQbx7OlhZvPo914XbU92m4cz/rq294hsJ6L28F21u+x9Vys/xbM4yVpfnf3U4cgFYKJkoEOAQAAYFMhAABOXu0QsCAAAMDBqwsCLhkAAAA6BAAAONEhAAAAnkWHAAAAh6hHOwQsCAAAcOCSAQAA8Cw6BAAAOHi1Q9CvC4K3fvZ6jzVHzraZxho/3PZGKzd9ZZSp7ku3JX5byM9ddt0EU93Qidea6iQpMO5qU10kM9dUF82wvYlPS4ctL/Scuc6W0WutO9PSYatrtdW1tDUY62z/GXzabqs7Z6xrs0YmJ6nOHulsq7PG39rPw/Z1th5XkjqsMdEu11m5fVxrXcwaXezy+d59ne3/QFwcOgQAADjEjO+zMdiwIAAAwMGrlwzYVAgAAOgQAADgRIcAAAB4Fh0CAAAc6BAAAADPokMAAIBDLGrLvhhsWBAAAODAJQMAAOBZ/doh2HmypceaOy+/1DTWDQtuNdVlz5plqtNXbjGVncnIMtUdamm3HVfS0cZzprqP/njGVNfQcspUd8oYE93Q0mqqa/rUds7nz9nqWs/ZIolbjeO1GcdrM0Yht7favn7RDludNR3NOp61LmKdn/FVU7Td5fHM52v/mbMyz9FY53YCnnV+br/iTdZx9X9vc3e8C6BDAAAAPIs9BAAAOFg7PoMNCwIAABy8+uZGXDIAAAB0CAAAcGJTIQAA8Cw6BAAAOHi1Q8CCAAAAB68uCLhkAAAA6BAAAODk1Q6BLxaLxZI9CQAAkFxcMgAAACwIAAAACwIAACAWBAAAQCwIAACAWBAAAACxIAAAAGJBAAAAxIIAAACIBQEAABDvZdArU6dOVXp6ujIyMiRJP/jBDzRlypQkz6pn5eXleuWVV3T8+HFVVlZq4sSJkqSjR4+qtLRUTU1NyszMVHl5ufLz85M72R5c6FwG0vemsbFRS5cu1bFjx5Senq4vf/nLWr16tbKysvT73/9eK1asUGtrq8aOHat/+Zd/0ejRo5M95QtKdC5XXXWVJk6cKL//s9cd69at01VXXZXkGV/YokWLVFNTI7/fr2HDhunhhx9WOBwecD8nFzqPgfQzgiSJwewb3/hG7NChQ8meRq+9+eabsRMnTnSZ/7333hv79a9/HYvFYrFf//rXsXvvvTdZUzS70LkMpO9NY2Nj7He/+13nn9euXRv70Y9+FItEIrHp06fH3nzzzVgsFott3rw5VlpamqxpmlzoXGKxWGzixImxlpaWZE2t15qbmzs//q//+q9YUVFRLBYbeD8nFzqPgfQzguTgkoEH3HTTTQqFQnHPnTp1StXV1SooKJAkFRQUqLq6Wg0NDcmYoll35zLQZGZm6pZbbun889e+9jWdOHFC7733njIyMnTTTTdJku655x7t2rUrWdM0udC5DEQjRozo/LilpUU+n29A/px0dx6ABZcMeukHP/iBYrGYbrzxRn3/+99XMBhM9pQuSm1trXJychQIBCRJgUBA2dnZqq2tVVZWVpJnd3EG4vcmGo3ql7/8paZOnara2lrl5uZ2fi4rK0vRaLSzVZ3qnOfyuXvvvVeRSES33XabSkpKlJ6ensQZ9uzHP/6x9u3bp1gspn//938fsD8nf30enxuIPyPoP3QIeuHpp5/Wiy++qOeff16xWEyrV69O9pTwZwP1e/PII49o2LBh+va3v53sqXxhf30uv/nNb/Sf//mfevrpp/XRRx9p8+bNSZ5hz37yk5/oN7/5jb73ve9p3bp1yZ7ORevuPAbqzwj6DwuCXvi8VZ2enq7i4mK99dZbSZ7RxQuFQqqrq1MkEpEkRSIR1dfXD9h2/ED83pSXl+sPf/iDfvrTn8rv9ysUCsW12xsaGuT3+wdEd+Cvz0X6y/fk0ksv1dy5cwfE9+RzRUVFeuONN3T55ZcP6J+Tz8+jsbFxQP6MoH+xIDD69NNPdebMGUlSLBbTzp07FQ6Hkzyrizd69GiFw2FVVVVJkqqqqhQOh1O6DXohA/F7s379er333nvavHlzZxv92muv1fnz5/U///M/kqT/+I//0KxZs5I5TZPuzuX06dM6f/68JKmjo0OvvPJKSn9Pzp49q9ra2s4/7927VyNHjhxwPycXOo+MjIwB9zOC/ueLxWKxZE9iIPjjH/+okpISRSIRRaNRjR8/XsuXL1d2dnayp9ajNWvWaPfu3frkk080atQoZWZm6qWXXtKRI0dUWlqq5uZmBYNBlZeX68orr0z2dBPq7ly2bt06oL43hw8fVkFBgfLz83XJJZdIkvLy8rR582a99dZbKisri7vt8LLLLkvyjC/sQudy//33a8WKFfL5fOro6ND111+vhx56SMOHD0/yjLv3ySefaNGiRTp37pz8fr9GjhypZcuW6ZprrhlQPycXOo9gMDigfkaQHCwIAAAAlwwAAAALAgAAIBYEAABALAgAAIBYEAAAALEgAAAAYkEAAAAk/f84KjxCLvf38QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ak-XZguCZ7N4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "d4555f07-c47d-45a7-abe6-814af495dec3"
      },
      "source": [
        "wandb.init(entity='authors', project='loss-landscape', id='smallcnn_same_model_investigations')\n",
        "wandb.log({'prediction_disagreement': wandb.Image('prediction_disagreement.png')})\n",
        "wandb.log({'functional_similarity': wandb.Image('functional_similarity.png')})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/loss-landscape\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/loss-landscape/runs/smallcnn_same_model_investigations\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape/runs/smallcnn_same_model_investigations</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TGzy-ibgmv9i",
        "colab_type": "text"
      },
      "source": [
        "# Similarity of Functions Across Randomly Initialized Trajectories"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nyZcTvtSnAnM",
        "colab_type": "text"
      },
      "source": [
        "## Disagreement of predictions\n",
        "\n",
        "(Refer figure 3(a) from paper)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "jlknaYiNcJXE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "4637d6fd843045489c4ad210d0ee89e6",
            "8ae82964608a44edb8f5e268a4046061",
            "184dc95f2fa3415d8b72538ffd11ed0f",
            "bbcdf9fa67e44a798eaa2516914deaee",
            "60d2d2edd2944492b89ff6c9662b9fea",
            "c544d2cfbe76463eb3ece0e3424b6d95",
            "61068162282549e5b5294e30f76e7f84",
            "2a7154471f29417c97196e2ff2601436"
          ]
        },
        "outputId": "78513963-604e-457a-c674-93092220cf8f"
      },
      "source": [
        "predictions = []\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model(INDEPENDENT_MODEL_PATH+independent_models[i])\n",
        "  # get predictions for model\n",
        "  _, preds = evaluate(testloader, model)\n",
        "\n",
        "  predictions.append(preds)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4637d6fd843045489c4ad210d0ee89e6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Q12GUXQJcJXS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "6127015f31134069ac03cfa32aaf292b",
            "b3be6995478148de8f9ebe9cc4e38ab4",
            "67a8a5c7ff874f11ae52135ff605588d",
            "77800b0a1af04b4681d6422271fde7b2",
            "a695b28df7824d918cf6ba98385c520f",
            "8ba9e041a5714de3aae213ffd1e429e7",
            "9e4500b363d04d5cb2cb4d71971b2501",
            "9a0a947f02f4436eb102b2aa12d2d463"
          ]
        },
        "outputId": "2bce7e91-22d9-438b-a72f-a932bf2fdd0e"
      },
      "source": [
        "empty_arr = np.zeros(shape=(10,10))\n",
        "\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  preds1 = predictions[i]\n",
        "  for j in range(i, 10):\n",
        "    preds2 = predictions[j]\n",
        "    \n",
        "    # compute dissimilarity\n",
        "    dissimilarity_score = 1-np.sum(np.equal(preds1, preds2))/10000 \n",
        "    \n",
        "    empty_arr[i][j] = dissimilarity_score\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = dissimilarity_score\n",
        "\n",
        "dissimilarity_coeff = empty_arr[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6127015f31134069ac03cfa32aaf292b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8XHgFIrNcJXb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "a4367475-62d6-4910-989a-d2b8a53d5651"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns.heatmap(dissimilarity_coeff, cmap='RdBu_r');\n",
        "plt.xticks([0,2,4,6,8],[0,2,4,6,8]);\n",
        "plt.yticks([0,2,4,6,8],[8,6,4,2,1]);\n",
        "\n",
        "plt.savefig('independent_prediction_disagreement.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAHYCAYAAAAyFvBfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3BU9f3/8dfuJpHbbCFIwnJRLn6hUYiK/cHXUWSkQGgnGrBD0ZSWjjZoNUHBC9HaXFSs4dtBkYC2jlD4oZYfYwdKTBEpTmuwv6J+8UZQEZNyS0JNiCuU6+7+/ui3+X22S06Ize7Z9PN8zOxMOOckn/cBkn3n9fmcczyRSCQiAABgNa/bBQAAAPfREAAAABoCAABAQwAAAERDAAAAREMAAABEQwAAANTJhuDEiRP68MMPFQwG41UPAABwgWND8Nprr2ncuHGaPn263n//fX3729/WAw88oKlTp2r79u2JqhEAAMRZitPOFStW6KWXXlIwGFRBQYGeeeYZjRs3Tvv27dO9996ryZMnd2qwsp6X/EvFxtPI3qlul9ChfcfPuF2Co0E9HP87ue7wybNul+DIn5LcM3g9fcldX3dw7GzY7RIc/eQ/rnW7BEend61KyDh3eIZ1+dd8NlLf5V+zq3X4E3z06NGSpN69e2vcuHGSpJEjR8a3KgAAkFCOLb/H49G+ffu0a9cu/e1vf9O7774rSaqrq1MoFEpIgQAAIP4cE4L58+frlltukdfr1ZNPPqlly5bpr3/9qxobG1VWVpagEgEASByfx+0K3OHYEFx//fXauXNn25/Hjx+vPXv2aODAgbrwwgvjXhwAAInm89jZEXRqFZjP59OYMWPiVQsAAHBJci8LBwAgwWydMuA6IgAAQEIAAICJNQQAAIApAwAAYC8SAgAADLZOGZAQAAAAEgIAAEysIQAAANYiIQAAwGDrGgIaAgAADLZG57aeNwAAMJAQAABgsHXKgIQAAACQEAAAYLL1skMaAgAADEwZAAAAa5EQAABgsHXKgIQAAACQEAAAYLJ1DQENAQAABqYMAACAtUgIAAAw2DplQEIAAABICAAAMNm6hoCGAAAAg60NAVMGAACAhAAAABOLCgEAgLVICAAAMLCGAAAAWIuEAAAAg61rCGgIAAAwMGUAAACsRUIAAIDB1ikDEgIAAEBCAACAydY1BAltCE6EwokcrlNaTidvbf+QmuQx1ulwxO0SHCV7fcGzyf1/MNlj1GT/95Wkn/zHtW6X4Gjx3hq3S0gKyf5/PV6YMgAAAEwZAABg8pIQAAAAW5EQAABg8Fi6qpCGAAAAg9fShoApAwAAQEIAAIDJ47Pzd2U7zxoAAEQhIQAAwJAMiwrr6upUXFys1tZW9e3bVxUVFRo2bFjUMStWrFB1dbW8Xq9SU1O1YMECTZw4UZJUXFysN998U/369ZMkTZ8+XT/+8Y8dx6QhAAAgyZSWlio/P195eXnatGmTSkpKtHbt2qhjsrOzdeutt6pnz5766KOPNGfOHNXU1KhHjx6SpHnz5mnOnDnnPSYNAQAAhnhcZRAMBhUMBmO2+/1++f3+qG3Nzc2qra3V6tWrJUm5ubl69NFH1dLSovT09Lbj/pEGSNLo0aMViUTU2tqqgQMHfqUaaQgAADB4vF2/vG7NmjWqrKyM2V5YWKiioqKobQ0NDcrMzJTP55Mk+Xw+ZWRkqKGhIaohMG3cuFEXXXRRVDOwevVqrV+/XkOHDtW9996rkSNHOtZIQwAAQJzNnTtXM2fOjNn+z+nAV7Fz504tW7ZMq1atatu2YMECDRgwQF6vVxs3btSPfvQjbdu2ra3JOBcaAgAADPGYMjjX1EB7AoGAmpqaFAqF5PP5FAqFdOTIEQUCgZhjd+3apfvvv18rV67UiBEj2rZnZma2fTxjxgz97Gc/U2NjowYPHtzuuFx2CABAEunfv7+ysrJUVVUlSaqqqlJWVlbMdMH777+vBQsW6Omnn9Zll10Wta+pqant4zfeeENerzeqSTgXEgIAAAzJcNlhWVmZiouLtXLlSvn9flVUVEiSCgoKNH/+fI0dO1bl5eU6efKkSkpK2j5vyZIlGj16tBYtWqTm5mZ5PB716dNHzzzzjFJSnN/yaQgAADAkw50KR44cqQ0bNsRsf+6559o+fvnll9v9/F/96ledHtP9swYAAK4jIQAAwMDTDgEAgLVICAAAMHi8diYENAQAABi8SbCo0A12njUAAIhCQgAAgCEZ7kPgBhICAABAQgAAgImEAAAAWIuEAAAAg61XGdAQAABgYMoAAABYi4QAAACD19I7FZIQAAAAEgIAAEweFhUCAAAefwwAAKxFQgAAgIHLDgEAgLVICAAAMLCoEAAAsKgQAADYi4QAAACDhzsVduz48ePavXu3jh07Fq96AACACxwbgpKSErW0tEiS3nnnHU2dOlUPPPCApk6dqpqamoQUCABAInl93i5/dQeOUwbvvvuu0tPTJUnLli3Ts88+q+zsbNXV1enee+/Vtdde26nBxvgv+OqVxpnPk/wR0RdnQm6X4Cg9zed2CY5OhyNul+Dokj5pbpfgqDZ4yu0SHJWMmuh2CR1avDe5f5Hqk9I93rgQH44NwalT//8HwPHjx5WdnS1JGj58uM6cORPfygAAcAE3JjqHq6++Wk888YROnDihCRMmqLq6WpK0Y8cO9e3bNyEFAgCQSB6ft8tf3YFjlQ899JDOnj2r6667Tq+99poWLlyoMWPGaNWqVXr88ccTVSMAAIgzxymDtLQ0Pfzww1q4cKH279+vcDisQCCgfv36Jao+AAASyuPtHr/Rd7Xzug9Br1699PWvfz3etQAAAJdwYyIAAAzd5TLBrkZDAACAobssAuxqdp41AACIQkIAAICBhAAAAFiLhAAAAAOXHQIAAHl8yf1clnixsw0CAABRSAgAADCwqBAAAFiLhAAAAIPX0kWFdp41AACIQkIAAIDB1jUENAQAABhsbQjsPGsAABCFhAAAAIOtdyq086wBAEAUEgIAAAy2riGgIQAAwGBrQ2DnWQMAgCgkBAAAGLwkBAAAwFYkBAAAGGy97JCGAAAAA4sKAQCAtUgIAAAwkBAAAABrkRAAAGCwdVGhnWcNAACikBAAAGDw+nxul+AKGgIAAAwsKgQAANYiIQAAwEBCAAAArEVCAACAwdbLDmkIAAAwJMOUQV1dnYqLi9Xa2qq+ffuqoqJCw4YNizpmxYoVqq6ultfrVWpqqhYsWKCJEydKkk6cOKEHH3xQu3fvls/n06JFi3T99dc7jklDAABAkiktLVV+fr7y8vK0adMmlZSUaO3atVHHZGdn69Zbb1XPnj310Ucfac6cOaqpqVGPHj30/PPPq0+fPnrttddUX1+v733ve9q6dat69+7d7pgJbQg+OXY6kcN1is/jcbuEDoUiEbdLcHTsbNjtEhydDif331/d8TNul+CoZNREt0tw9Mgnb7hdQofSkuA3T3QsHglBMBhUMBiM2e73++X3+6O2NTc3q7a2VqtXr5Yk5ebm6tFHH1VLS4vS09PbjvtHGiBJo0ePViQSUWtrqwYOHKjf/e53euKJJyRJw4YN05gxY/THP/5R3/rWt9qtkYQAAIA4W7NmjSorK2O2FxYWqqioKGpbQ0ODMjMz5fufGyT5fD5lZGSooaEhqiEwbdy4URdddJEGDhwoSTp8+LAGDx7ctj8QCKixsdGxRhoCAAAM8VhUOHfuXM2cOTNm+z+nA1/Fzp07tWzZMq1atepf+jo0BAAAGDzerr918bmmBtoTCATU1NSkUCgkn8+nUCikI0eOKBAIxBy7a9cu3X///Vq5cqVGjBjRtn3QoEE6dOhQW6LQ0NCgCRMmOI7LhBYAAEmkf//+ysrKUlVVlSSpqqpKWVlZMdMF77//vhYsWKCnn35al112WdS+6dOna/369ZKk+vp6ffDBB1FrDs6FhgAAAJPX1/WvTiorK9O6deuUk5OjdevWqby8XJJUUFCgDz74QJJUXl6ukydPqqSkRHl5ecrLy9PHH38sSbrtttsUDAY1depU3X777XrkkUfUp08fxzGZMgAAIMmMHDlSGzZsiNn+3HPPtX388ssvt/v5vXr10tNPP92pMWkIAAAwWXqnQjvPGgAARCEhAADA4PF1/VUG3QENAQAApjhcdtgdMGUAAABICAAAiEJCAAAAbEVCAACAIR7PMugOaAgAADAxZQAAAGxFQgAAgImEAAAA2IqEAAAAA4sKAQAAUwYAAMBeJAQAAJhICAAAgK1ICAAAMNj6+GMSAgAAQEIAAEAULjsEAAAsKgQAANYiIQAAwOAhIQAAALYiIQAAwMSiQgAAwJQBAACwFgkBAAAmEgIAAGArEgIAAEwsKgQAADzcCAAAWIuEAAAAE4sKAQCArUgIAAAwkRCcnzfffDMedQAAkBQ8Xm+Xv7oDx4Tg008/jdn24IMPatWqVYpEIrrkkkviVhgAAEgcx4YgNzdXgwcPViQSadv2+eefq6CgQB6PR7///e87NdiFack7Q5Hm9bhdQod8SV7i11KTuwtO9vq+P/g/3S7B0X/t2+F2CY6G+y9wu4QO/XfrSbdLcORPSe7vkYSxdMrA8R26sLBQ7733nsrLyzVo0CBJ0uTJk7V9+/aEFAcAABKjw4agtrZWCxcuVF5enm655RZ5PEn+ayoAAP8Kj51JSYdnfemll2rt2rU6dOiQfvjDH+rMmTOJqAsAACTQeU3qp6Wl6b777tO7776rnTt3xrsmAADcY2lC0KlVfldccYWuuOKKeNUCAIDrIpY2BHaeNQAAiJK81wECAOAGEgIAAGArEgIAAEyWXl5PQwAAgKmbPHugq9l51gAAIAoJAQAABi47BAAA1iIhAADAZGlCQEMAAIDJ0obAzrMGAABRSAgAADCREAAAAFuREAAAYOCyQwAAYC0SAgAATJYmBDQEAACYLH24kZ1tEAAAiEJCAACAydIpAzvPGgAARCEhAADAYOtlhzQEAACYvHY2BHaeNQAAiEJCAACAydIpAzvPGgCAJFZXV6fZs2crJydHs2fPVn19fcwxNTU1uummmzRmzBhVVFRE7Vu+fLmuvvpq5eXlKS8vT+Xl5R2OSUIAAIApCRKC0tJS5efnKy8vT5s2bVJJSYnWrl0bdczQoUO1ePFibdmyRadPn475GjNmzNCiRYvOe0z3zxoAgGTi8Xb5KxgM6uDBgzGvYDAYM3xzc7Nqa2uVm5srScrNzVVtba1aWlqijrv44ouVlZWllJSu+d2ehAAAgDhbs2aNKisrY7YXFhaqqKgoaltDQ4MyMzPl8/kkST6fTxkZGWpoaFB6evp5j/nKK6+opqZGAwYMUFFRka688krH42kIAAAwxOM+BHPnztXMmTNjtvv9/i4fS5Juvvlm3XHHHUpNTdWOHTt05513qrq6Wv369Wv3c2gIAACIM7/ff95v/oFAQE1NTQqFQvL5fAqFQjpy5IgCgcB5jzdgwIC2j6+55hoFAgHt3btX48ePb/dzWEMAAIApDmsIOqN///7KyspSVVWVJKmqqkpZWVmdmi5oampq+3jPnj06dOiQhg8f7vg5JAQAACSZsrIyFRcXa+XKlfL7/W2XFRYUFGj+/PkaO3as3n77bS1cuFDHjh1TJBLRK6+8osWLF2vixIlaunSpdu/eLa/Xq9TUVC1ZsiQqNTgXTyQSiSTi5CRpmX90oobqtDRv8j//2pfkJX4tlcDpX/H9wf/pdgmO/mvfDrdLcDS8d6rbJXTov1tPul2CI39Kcn8PL/zyk4SMc+rYF13+NS/o87Uu/5pdjYQAAABTEtyHwA12njUAAIhCQgAAgMHWxx/bedYAACBKQhOCZF641zPZV+xJ2v+3M26X4OjAiYStT/1KKrImuV2Co/996P+6XYKjulDY7RIcfXos9l7uyeZ0OLm/R4Jnk/vfOGEsTQiYMgAAwBDxJP8viPFgZxsEAACikBAAAGBI3N15kgsJAQAAICEAAMAUtjQioCEAAMBgZzvAlAEAABAJAQAAUZL8dhFxQ0IAAABICAAAMEUsXVRIQgAAAEgIAAAw2bqGgIYAAACDpf0AUwYAAICEAACAKLZOGZAQAAAAEgIAAEy2XnZIQwAAgCHsdgEuYcoAAACQEAAAYLJ0xoCEAAAAkBAAABDF1ssOaQgAADDYepUBUwYAAICEAAAAE5cdAgAAa5EQAABgsHQJAQkBAAAgIQAAIErY0oiAhgAAAIOd7QBTBgAAQCQEAABEsfVOhSQEAACAhAAAAJOlawppCAAAMIUtXVbIlAEAACAhAADAZOuUAQkBAAAgIQAAwGTrZYc0BAAAGJgyAAAA1iIhAADAwGWH53D06FH95Cc/0a233qoXXnghal9RUVFcCwMAAInjmBCUlpZqyJAhmjRpkl566SX96U9/0lNPPaWUlBQdOHCg04OlJfEERagbTBqdTPKVLj+/dJLbJThatOcPbpfgzH+B2xU4Sk/zuV2Coz4pSfwD5n+keT1ul+Ao2etLlG7wdhAXjt9B9fX1euCBBzRt2jStWrVKAwYM0O23365Tp04lqj4AAJAAjg3BmTNn2j72eDwqLS3VqFGjNG/ePJoCAMC/pXAk0uWv7sCxIRg6dKjeeuutqG2LFi3S5Zdfrvr6+njWBQCAK0Lhrn91B45rCJYsWSKPJ3ZOaeHChbrxxhvjVhQAAEgsx4agb9++7e675JJLurwYAADc1l0i/q6W/MtyAQBA3HFjIgAADN3hMvR4oCEAAMDAlAEAALAWCQEAAIbucplgVyMhAAAAJAQAAJhsXUNAQwAAgMHWqwyYMgAAACQEAACYkvxJ83FDQgAAAEgIAAAwhSyNCEgIAAAADQEAAKZwJNLlr86qq6vT7NmzlZOTo9mzZ6u+vj7mmJqaGt10000aM2aMKioqovaFQiGVl5drypQpmjp1qjZs2NDhmDQEAAAYQpGuf3VWaWmp8vPz9eqrryo/P18lJSUxxwwdOlSLFy/WbbfdFrNv8+bN2r9/v7Zu3ar169dr+fLlOnjwoOOYNAQAAMRZMBjUwYMHY17BYDDm2ObmZtXW1io3N1eSlJubq9raWrW0tEQdd/HFFysrK0spKbHLAaurqzVr1ix5vV6lp6drypQp2rJli2ONLCoEAMAQjzsVrlmzRpWVlTHbCwsLVVRUFLWtoaFBmZmZ8vl8kiSfz6eMjAw1NDQoPT39vMZraGjQoEGD2v4cCATU2Njo+Dk0BAAAxNncuXM1c+bMmO1+v9+Fas6NhgAAAEM8Ljv0+/3n/eYfCATU1NSkUCgkn8+nUCikI0eOKBAInPd4gUBAhw8fVnZ2tqTYxOBcWEMAAIDB7asM+vfvr6ysLFVVVUmSqqqqlJWVdd7TBZI0ffp0bdiwQeFwWC0tLdq2bZtycnIcP4eGAACAJFNWVqZ169YpJydH69atU3l5uSSpoKBAH3zwgSTp7bff1nXXXafVq1fr17/+ta677jq98cYbkqS8vDwNGTJE06ZN03e/+13dddddGjp0qOOYnkgkcY91ej7964ka6t/SvuNn3C7B0c8vneR2CY4W7fmD2yU4GuO/wO0SHH1xJux2CY76pCT/7zctp0Nul+AozetxuwRH845+lJBx/s/7h7v8a3432zmuTwbJ/x0EAADijkWFAAAY4nHZYXdAQwAAgCHMw40AAICtSAgAADB8lWcP/DsgIQAAACQEAACYbF1USEIAAABICAAAMIUsTQhoCAAAMHDZIQAAsBYJAQAABi47BAAA1iIhAADAYOtlhzQEAAAYuMogAd46ejKRw3VKsj+LXpJ+fukkt0twdF/tH9wuwVFqkj/r/YszYbdLcHTgxBm3S3CU6knuf19J8iV5jWcsfSPE35EQAABgCHHZIQAAsBUJAQAABlsTAhoCAAAMtjYETBkAAAASAgAATCQEAADAWiQEAAAYbE0IaAgAADDY2hAwZQAAAEgIAAAwkRAAAABrkRAAAGAgIQAAANYiIQAAwGBrQkBDAACAwdaGgCkDAABAQgAAgOksCQEAALAVCQEAAAZb1xDQEAAAYLC1IWDKAAAAkBAAAGAKRUgIAACApUgIAAAw2LqGgIYAAACDrQ0BUwYAAICEAAAAEwkBAACwFgkBAACGUDjsdgmuICEAAAAkBAAAmGxdQ0BDAACAwdaGgCkDAABAQgAAgOksCQEAALAVCQEAAAZb1xDQEAAAYLC1IWDKAAAAkBAAAGAiIQAAANYiIQAAwGBrQkBDAACAgYagk2644QZt3ry5U58zxn/BVx0u7haOuMbtEjq09LMdbpfgqGfvVLdLcOTzeNwuwdGJUHL/EEr2v79L+qS5XUKH/noq5HYJjtLTmEW2mWND8Omnn7a77+jRo11eDAAAbouQEMTKzc3V4MGDFYnE/uW0trbGrSgAAJBYjg3B4MGD9eKLLyozMzNm36RJk+JWFAAAbglbmhA4ThhNmzZNhw4dOue+qVOnxqUgAACQeI4JwaJFi9rd9/DDD3d5MQAAuO1c0+Q24LJDAAAMti4q5BoTAABAQgAAgCkZFhXW1dWpuLhYra2t6tu3ryoqKjRs2LCoY0KhkB577DG98cYb8ng8mjdvnmbNmiVJWr58uV588UVlZGRIksaNG6fS0lLHMWkIAABIMqWlpcrPz1deXp42bdqkkpISrV27NuqYzZs3a//+/dq6dataW1s1Y8YMXX311RoyZIgkacaMGY5rAf8ZUwYAABgi4a5/BYNBHTx4MOYVDAZjxm9ublZtba1yc3Ml/f2eQLW1tWppaYk6rrq6WrNmzZLX61V6erqmTJmiLVu2fOXzJiEAAMAQj6sM1qxZo8rKypjthYWFKioqitrW0NCgzMxM+Xw+SZLP51NGRoYaGhqUnp4eddygQYPa/hwIBNTY2Nj251deeUU1NTUaMGCAioqKdOWVVzrWSEMAAECczZ07VzNnzozZ7vf74zLezTffrDvuuEOpqanasWOH7rzzTlVXV6tfv37tfg4NAQAAhngsKvT7/ef95h8IBNTU1KRQKCSfz6dQKKQjR44oEAjEHHf48GFlZ2dLik4MBgwY0HbcNddco0AgoL1792r8+PHtjssaAgAAkkj//v2VlZWlqqoqSVJVVZWysrKipgskafr06dqwYYPC4bBaWlq0bds25eTkSJKamprajtuzZ48OHTqk4cOHO45LQgAAgCEZbkxUVlam4uJirVy5Un6/XxUVFZKkgoICzZ8/X2PHjlVeXp7ee+89TZs2TZJ01113aejQoZKkpUuXavfu3fJ6vUpNTdWSJUuiUoNz8UQSeI/Gyq+NTtRQnbZwxDVul9ChpZ/tcLsERz19HrdLcOTzJHd9J0Lu/xBy0nTqrNslOBrVJ83tEjr011Mht0twlJ6W3KHx9z/fk5BxJjzyWpd/zT+XJP/zf5L7Xx8AACQEUwYAABjClj7ciIQAAACQEAAAYEqGRYVuICEAAAAkBAAAmGxNCGgIAAAwJMPjj93AlAEAACAhAADAlMD79SUVEgIAAEBCAACAKRJ2uwJ30BAAAGBgUSEAALAWCQEAAAZb70NAQgAAAEgIAAAw2ZoQ0BAAAGDg8ccAAMBaJAQAABhsnTIgIQAAACQEAACYSAgAAIC1SAgAADDYeutiGgIAAAw8/hgAAFiLhAAAAIOtiwoT2hAsHHFNIofrlKWf7XC7hA59GDzldgmO/le/Hm6X4OhEKLm/yU8n+Q+hUJLHqMlenyQdPnnG7RIcHT1DaGwzEgIAAAwsKgQAAIqEQ26X4AryIQAAQEIAAICJhAAAAFiLhAAAAIOtCQENAQAAhkjIzoaAKQMAAEBCAACAydYpAxICAABAQgAAgImEAAAAWIuEAAAAg60JAQ0BAAAGWxsCpgwAAAAJAQAAJhICAABgLRICAAAMYUsTAhoCAAAMTBkAAABrkRAAAGAgIQAAANYiIQAAwBAJ2ZkQ0BAAAGBgygAAAFiLhAAAAAMJAQAAsBYJAQAABhICAABgLRICAAAMkXDY7RJcQUMAAICBKQMAAGAtEgIAAAwkBAAAwFokBAAAGMKWJgQ0BAAAGGx9uBFTBgAAgIQAAAATiwoBAIC1SAgAADCQEAAAAEXCoS5/dVZdXZ1mz56tnJwczZ49W/X19THHhEIhlZeXa8qUKZo6dao2bNhwXvvaQ0MAAECSKS0tVX5+vl599VXl5+erpKQk5pjNmzdr//792rp1q9avX6/ly5fr4MGDHe5rDw0BAACGeCQEwWBQBw8ejHkFg8GY8Zubm1VbW6vc3FxJUm5urmpra9XS0hJ1XHV1tWbNmiWv16v09HRNmTJFW7Zs6XBfexK6huD0rlWJHA4AgE6Lx3vV8uXLVVlZGbO9sLBQRUVFUdsaGhqUmZkpn88nSfL5fMrIyFBDQ4PS09Ojjhs0aFDbnwOBgBobGzvc1x4WFQIAEGdz587VzJkzY7b7/X4Xqjk3GgIAAOLM7/ef95t/IBBQU1OTQqGQfD6fQqGQjhw5okAgEHPc4cOHlZ2dLSk6FXDa1x7WEAAAkET69++vrKwsVVVVSZKqqqqUlZUVNV0gSdOnT9eGDRsUDofV0tKibdu2KScnp8N97fFEIpFIfE4JAAB8Ffv27VNxcbGCwaD8fr8qKio0YsQIFRQUaP78+Ro7dqxCoZAeeeQR7dixQ5JUUFCg2bNnS5LjvvbQEAAAAKYMAAAADQEAABANAQAAEA0BAAAQDQEAAFCCGoLzeWoTzu3o0aMqKChQTk6ObrjhBhUWFsbczxrnp7KyUqNHj9Ynn3zidindzqlTp1RaWqpp06bphhtu0E9/+lO3S+pWXn/9dc2YMUN5eXm68cYbtXXrVrdLAmIk5LLDH/zgB/rOd76jvLw8bdq0SS+//LLWrl0b72H/LbS2turjjz/WhAkTJEkVFRX64osv9Pjjj7tcWfeye/duPfnkk/rss8/07LPPatSoUW6X1K089thj8nq9evDBB+XxePT555/rwgsvdLusbiESiWj8+PF64YUXNGrUKH300Ue65ZZb9M4778jrJaRF8oj7/8bzfWoTzq1v375tzYAkXXHFFTp8+LCLFXU/p0+f1iOPPKKysjK3S+mWjrQsVSoAAAJUSURBVB8/ro0bN+ruu++Wx+ORJJqBTvJ6vfryyy8lSV9++aUyMjJoBpB04v4sg/N9ahM6Fg6H9dJLL2ny5Mlul9KtLFu2TDfeeKOGDBnidind0oEDB9S3b19VVlbqz3/+s3r37q27775b3/jGN9wurVvweDx66qmndOedd6pXr146fvy4fvnLX7pdFhCDFrUbefTRR9WrVy/NmTPH7VK6jV27dunDDz9Ufn6+26V0W6FQSAcOHNCll16q3/zmN7rvvvtUVFSkY8eOuV1at3D27Fn94he/0MqVK/X666/rmWee0T333KPjx4+7XRoQJe4NgfnUJkntPrUJzioqKvSXv/xFTz31FFFjJ7z11lvat2+fvvnNb2ry5MlqbGzUbbfdppqaGrdL6zYCgYBSUlLapv0uv/xy9evXT3V1dS5X1j3s2bNHR44c0VVXXSVJuuqqq9SzZ0/t27fP5cqAaHF/ZznfpzahfUuXLtWHH36oFStWKC0tze1yupV58+appqZG27dv1/bt2zVw4EA9//zzuvbaa90urdtIT0/XhAkT2h6SUldXp+bmZl188cUuV9Y9DBw4UI2Njfrss88k/f2hNc3NzbroootcrgyIlpCrDNp7ahM6tnfvXuXm5mrYsGHq0aOHJGnIkCFasWKFy5V1T5MnT+Yqg6/gwIEDeuihh9Ta2qqUlBTdc889mjRpkttldRu//e1v9dxzz7Utypw/f76mTJniclVANJ52CAAAWFQIAABoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAICk/wevx4tDUX5HFQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I8vAgQDAfZTr"
      },
      "source": [
        "## Cosine Similarity of weights\n",
        "\n",
        "(Refer figure 3(a) from paper)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vVJxb0fLfZTv",
        "colab": {}
      },
      "source": [
        "# Get the weights of the input model. \n",
        "def get_model_weights(model):\n",
        "  model_weights = []\n",
        "  # iterate through model layers.\n",
        "  for layer in model.layers:\n",
        "    # grab weights of that layer\n",
        "    weights = layer.get_weights() # list\n",
        "    # check if layer got triainable weights\n",
        "    if len(weights)==0:\n",
        "      continue\n",
        "\n",
        "    # discard biases term, wrap with ndarray, flatten weights\n",
        "    model_weights.extend(np.array(weights[0]).flatten())\n",
        "\n",
        "  return np.array(model_weights)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tOFUvSa0fZT9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "218521c71fc3446f84e98a8f4ce58ce3",
            "506668264965452c928dc5e79f977c10",
            "33ad336ed5d24cda83ccefca6ec6fabe",
            "ab14831d6979459fa56f9a5020c87c39",
            "4bd35e14c57943b38c0823c5939e387f",
            "7e71e4325049484bb997aa508bfa251f",
            "de958c9039cf498892218990dc29139c",
            "035e47e2de8345b7935b7c6845803132"
          ]
        },
        "outputId": "f5f65374-8509-45fc-a49a-afd78d1e190f"
      },
      "source": [
        "weights_of_models = []\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  # load model\n",
        "  model = tf.keras.models.load_model(INDEPENDENT_MODEL_PATH+independent_models[i])\n",
        "  # get predictions for model\n",
        "  weights = get_model_weights(model)\n",
        "\n",
        "  weights_of_models.append(weights)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "218521c71fc3446f84e98a8f4ce58ce3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "x6BtmhI0fZUL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "e6352df87c7541d19b45e3726e17d1a0",
            "e88a5dda7dcb41469e836908f626b452",
            "471d89ea16f04055ba49485307ef9ed8",
            "a713e10e41464f6cb4a3dfecdd7a3eea",
            "752af4f19b1a42e094bbfa539670fb1a",
            "a49bbe7a7a194dc78aaca78e984174a6",
            "642bc68664bb4b5ca4d3f7049ac6d204",
            "03ef3c15f9704cf6bd7fd1df3fe3dfd1"
          ]
        },
        "outputId": "03d7362b-5275-4d1e-e7ae-24a58810f846"
      },
      "source": [
        "empty_arr = np.zeros(shape=(10,10))\n",
        "\n",
        "for i in tqdm_notebook(range(10)):\n",
        "  weights1 = weights_of_models[i]\n",
        "  for j in range(i, 10):\n",
        "    weights2 = weights_of_models[j]\n",
        "    \n",
        "    # compute cosine similarity of weights\n",
        "    cos_sim = np.dot(weights1, weights2)/(norm(weights1)*norm(weights2))\n",
        "    \n",
        "    empty_arr[i][j] = cos_sim\n",
        "    if i is not j:\n",
        "      empty_arr[j][i] = cos_sim\n",
        "\n",
        "cos_sim_coeff = empty_arr[::-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e6352df87c7541d19b45e3726e17d1a0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Gih-HmwHfZUU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "5231047c-60e9-4335-f05e-f02031e2f90e"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "sns.heatmap(cos_sim_coeff, cmap='RdBu_r');\n",
        "plt.xticks([0,2,4,6,8],[0,2,4,6,8]);\n",
        "plt.yticks([0,2,4,6,8],[8,6,4,2,1]);\n",
        "\n",
        "plt.savefig('independent_functional_similarity.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAHYCAYAAABHrTENAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAf9ElEQVR4nO3df5CV9X0v8M+egxtB3SIouIgJEUfcZIKOZsJtR8eEX8t0VlemNzKuTkltsLcRTKKpYsbwo8mk3Uyn0YjRxltTHIzXy01H45ZRkmLjJU00dWLEbDQVoUJcIC6QVVAXzp77R1rmu3fhwAq7z57nvF4zZwae/e5zPo+T8Dnv7/P9PqeuXC6XAwDIvULWBQAAw0PTB4AaoekDQI3Q9AGgRmj6AFAjNH0AqBGaPgCMIO3t7TFz5syYNm1a/OpXvzrsmFKpFCtXrozZs2fHnDlzYu3atcd07kE1/bfffjtefPHF6OnpGcyvAQDHaNasWfHQQw/F2WeffcQxjz/+eLz22muxfv36eOSRR+Luu++O7du3H/XcFZv+97///bj44otj3rx58cILL8Qf/uEfxq233hpz5syJDRs2DP5KAKAG9fT0xPbt2we8DheiP/rRj0ZjY2PF861bty4++clPRqFQiHHjxsXs2bPjiSeeOGodoyr98J577omHH344enp6YtGiRXHvvffGxRdfHJs3b45bbrklZs6cedQ3SDV8/LZBjR9OJ41pyLqEozr4zr6sS6jowNtvZV1CRe877fSsS6io1PtO1iVU1Lvvt1mXUFH9Kb+XdQlVr+2Hj2RdQkX3lbcOy/v8j7opJ/ycH/7GLbFq1aoBxxcvXhxLliwZ9Pm6urpi0qRJh/7e2NgYO3bsOOrvVWz6ERHTpk2LiIhTTjklLr744oiImDp16qALBIBatXDhwpg/f/6A4w0Nwxs4Kzb9urq62Lx5c/T09MT+/fvj+eefj4suuii2bNkSpVJpuGoEgKrW0NBwQht8Y2NjvP766zF9+vSIGJj8j6Ri07/pppvimmuuiUKhEF//+tfjrrvuit/85jexY8eOWLFixQkpHABGkmJd1hUc3bx582Lt2rUxd+7c2Lt3b/zgBz+Ihx566Ki/V7Hpf+ITn4hnn3320N8/9rGPxS9/+cs466yz4owzzjj+qgFghCnWZdv1v/KVr8T69evjjTfeiD/5kz+JsWPHxj/90z/FokWL4qabboqPfOQj0draGj//+c9j7ty5ERFx4403xjnnnHPUc9cN51frWsh3fCzkOz4W8h0fC/nyz0K+37mp+METfs5vlLac8HO+F0ddyAcAtaQapvffK0/kA4AaIekDQCLre/pDSdMHgITpfQCg6kn6AJDI8/S+pA8ANULSB4CEe/oAQNWT9AEgked7+po+ACTyPAWe52sDABKSPgAk8jy9L+kDQI2Q9AEgkecte5o+ACRM7wMAVU/SB4BEnqf3JX0AqBGSPgAk8nxPX9MHgITpfQCg6kn6AJDI8/S+pA8ANULSB4BEnu/pa/oAkMhz0ze9DwA1QtIHgISFfABA1ZP0ASDhnj4AUPUkfQBI5PmevqYPAAnT+wBA1ZP0ASCR5+l9SR8AaoSkDwCJPN/TH9amP+rkU4bz7Qal72Bv1iVUvVMmnJN1CRXt73496xIqqh/TkHUJFY0Zwf//jYgo9b6TdQlHdc1T38m6hIq+c/mCrEuo6L5heh/T+wBA1TO9DwCJgqQPAFQ7SR8AEnU5Xsmn6QNAopDjpm96HwBqhKQPAIm6Yn7zcH6vDADoR9IHgESeF/JJ+gBQIyR9AEjkefW+pg8AibpCfifB83tlAEA/kj4AJPI8vS/pA0CNkPQBIJHnLXuaPgAkPJEPAKh6kj4AJCzkAwCqnqQPAIm6Qn6TvqYPAImChXwAQLWT9AEgked9+pI+ANQISR8AEpI+AFD1JH0ASOR59b6mDwAJ0/sAQNWT9AEgUcjxE/kkfQCoEZI+ACTqLOQDgNrgq3UBgKon6QNAwpY9AKDqSfoAkLCQDwBqxEhYyLdly5ZYunRp7N27N8aOHRvt7e0xZcqUfmO6u7vj9ttvj66urjh48GDMmDEj7rjjjhg16sitPb8fZwCgSi1fvjza2triySefjLa2tli2bNmAMffdd19MnTo1Hn/88fje974Xv/jFL2L9+vUVz6vpA0CirlB3wl+D0d3dHZ2dndHS0hIRES0tLdHZ2Rm7d+/uX2ddXezbty/6+vqit7c3Dhw4EBMnTqx47kFN7+/bty+2bt0aH/jAB+LUU08d1EUAQK3q6emJnp6eAccbGhqioaGh37Gurq6YOHFiFIvFiIgoFosxYcKE6OrqinHjxh0a95nPfCaWLFkSl156abz99ttx7bXXxiWXXFKxjopJf9myZYc+WTz33HMxZ86cuPXWW2POnDmxcePGY7tSAKgihWLhhL9Wr14ds2bNGvBavXr1e67ziSeeiGnTpsXGjRvj6aefjn/7t3+LJ554ouLvVEz6zz///KFPFXfddVfcd999MX369NiyZUvccsstcemllw6qwHJfaVDjh1PfwQNZl3BUdYWRfTem9809WZdQ0Zjxk7IuoaKR/t/v7T3bsi6houuf35B1CUf18Cfasi6hshH8b3S1W7hwYcyfP3/A8f8/5UdENDY2xs6dO6NUKkWxWIxSqRS7du2KxsbGfuPWrFkTX/3qV6NQKMRpp50WM2fOjGeeeSbmzZt3xDoqNv1333330J/37dsX06dPj4iID37wg3HgwMhvkgAwWEPxcJ7DTeMfyfjx46OpqSk6OjqitbU1Ojo6oqmpqd/UfkTE5MmT4+mnn47p06dHb29v/PjHP445c+ZUPHfF6Pj7v//78dd//dfx9ttvx4wZM2LdunUREfGjH/0oxo4de0zFA0A1qSsWTvhrsFasWBFr1qyJ5ubmWLNmTaxcuTIiIhYtWhSbNm2KiIgvfvGL8dxzz8UVV1wRV111VUyZMiWuvvrqytdWLpfLR/phb29vfO1rX4vHHnssxo4dG9u2bYtRo0bFjBkzYsWKFXHOOecM6iJOn7t8UOOHk+n941fu68u6hIrqTzs96xIqGvnT+zuyLqEi0/vHbyTfgo2I6PmX9mF5n5c+fdUJP+cF//PRE37O96Li9H59fX3ccccdcfPNN8drr70WfX190djYGKefPrL/8QSA92qkB6zjcUxb9saMGRMXXHDBUNcCAAwhj+EFgETBs/cBoDbk+Qt38ntlAEA/kj4AJCR9AKDqSfoAkKj5LXsAUCvq/vPb7fIovx9nAIB+JH0ASFjIBwBUPUkfABKFHC/ky++VAQD9SPoAkMjzPX1NHwASeW76+b0yAKAfSR8AEnl+Il9+rwwA6EfSB4BEnu/pa/oAkMhz08/vlQEA/Uj6AJAoSPoAQLWT9AEgkecte5o+ACQs5AMAqp6kDwAJSR8AqHqSPgAk8ryQL79XBgD0I+kDQKJQLGZdwpDR9AEgYSEfAFD1JH0ASEj6AEDVk/QBIJHnLXuaPgAkTO8DAFVvWJN+qfed4Xy73Ok72Jt1CVWt9809WZdQUWHUSVmXUNH1z2/IuoSKHrhoZtYlHNWp9SdnXUJF+7tfz7qEEUHSBwCqnnv6AJCwkA8AakRdIb+P4c3vxxkAoB9JHwBSkj4AUO0kfQBI5XghX36vDADoR9IHgERdMb/39DV9AEhZyAcAVDtJHwBSkj4AUO0kfQBIePY+ANQK0/sAQLWT9AEgJekDANVO0geAhIV8AFArTO8DANVO0geAlKQPAFQ7SR8AEnn+al1JHwBqhKQPAClb9gCgRljIBwBUO0kfABJ1kj4AUO0kfQBIWcgHALXB9D4AUPUkfQBISfoAQLWT9AEgZSEfANSGkfCFO1u2bImlS5fG3r17Y+zYsdHe3h5TpkwZMG7dunVx7733Rrlcjrq6uvj2t78dZ5xxxhHPq+kDwAizfPnyaGtri9bW1njsscdi2bJl8eCDD/Ybs2nTpli1alWsXr06zjzzzHjzzTejvr6+4nnzO4cBAO9FoXjiX4PQ3d0dnZ2d0dLSEhERLS0t0dnZGbt37+437h/+4R/i+uuvjzPPPDMiIk477bR43/veV/Hckj4ADLGenp7o6ekZcLyhoSEaGhr6Hevq6oqJEydG8T9vMxSLxZgwYUJ0dXXFuHHjDo3bvHlzTJ48Oa699trYv39/zJkzJ/78z/886urqjliHpg8AqSHYsrd69epYtWrVgOOLFy+OJUuWvKdzlkqlePnll+Pb3/529Pb2xqc//emYNGlSXHXVVUf8nUE3/X/913+NP/iDP3hPBQLASFc3BKv3Fy5cGPPnzx9w/P9P+RERjY2NsXPnziiVSlEsFqNUKsWuXbuisbGx37hJkybFvHnzor6+Purr62PWrFnxwgsvvPem/8orrww4dvvtt8cDDzwQ5XI5zjvvvEq/DgDE4afxj2T8+PHR1NQUHR0d0draGh0dHdHU1NRvaj/id/f6f/jDH0Zra2scPHgwfvKTn0Rzc3PFc9eVy+XykX54wQUXxNlnnx3pkJ07d8bEiROjrq4u/vmf//mYLuC/jJ5x46DG099Jo0/NuoSK3vd7R94mMhIcfPutrEuoaMH3V2ddQkWPzFmYdQkV9R08kHUJR1UYdVLWJVRU6n0n6xIq6vmX9mF5n75XfnLCz1k4778NavzmzZtj6dKl0dPTEw0NDdHe3h7nnntuLFq0KG666ab4yEc+En19fdHe3h5PP/10FAqFuPTSS+O2226LQoWZiopNf9WqVfHzn/88Vq5cGZMmTYqIiJkzZ8aGDRsGVfx/0fSPj6Z/fDT946PpHz9N//jUUtMfKhWn9xcvXhydnZ1x8803R2tra1xzzTUVVwUCQNWry+9u9qNe2Yc+9KF48MEH49e//nV86lOfigMHRv6naQBgoGNavV9fXx9f+MIX4vnnn49nn312qGsCgOzkOOkPasveRRddFBdddNFQ1QIAmSvnuOnn98oAgH48kQ8AUpI+AFDtJH0ASOV4a7qmDwCpIXj2/kiR3ysDAPqR9AEgYcseAFD1JH0ASOU46Wv6AJDKcdPP75UBAP1I+gCQkvQBgGon6QNAwpY9AKDqSfoAkMpx0tf0ASCV4y/cye/HGQCgH0kfAFI5nt7P75UBAP1I+gCQyPOWPU0fAFKF/Db9/F4ZANCPpA8AqRxP7+f3ygCAfiR9AEjlOOlr+gCQynHTz++VAQD9SPoAkMjzPv38XhkA0I+kDwApSR8AqHaSPgCk6uqyrmDIaPoAkDK9DwBUO0kfABK27AEAVW9Yk/7o0ycO59sNyrtv7cm6hKOqKxSzLqGid/bszLqEiq79v/8n6xIqemTOwqxLqKjv4IGsS6io/rTTsy7hqPZ3v551CRXVj2nIuoSRIcdJ3/Q+ACTKOV69n9+PMwBAP5I+ACTK5awrGDqSPgDUCEkfABJ9OY76mj4AJPLb8k3vA0DNkPQBINGX46gv6QNAjZD0ASBRzvFCPkkfAGqEpA8AiTzf09f0ASCR455veh8AaoWkDwCJPE/vS/oAUCMkfQBI5HnLnqYPAIm+rAsYQqb3AaBGSPoAkMjx7L6kDwC1QtIHgESet+xp+gCQyPPqfdP7AFAjJH0ASNiyBwBUPUkfABI5vqUv6QNArZD0ASDRl+Oor+kDQCK/Ld/0PgDUDEkfABJ5fiKfpA8ANULSB4BEjtfxSfoAkOqL8gl/DdaWLVtiwYIF0dzcHAsWLIitW7ceceyrr74aF154YbS3tx/1vJo+AIwwy5cvj7a2tnjyySejra0tli1bdthxpVIpli9fHrNnzz6m82r6AJAol0/8azC6u7ujs7MzWlpaIiKipaUlOjs7Y/fu3QPGfutb34qPf/zjMWXKlGM6t6YPAEOsp6cntm/fPuDV09MzYGxXV1dMnDgxisViREQUi8WYMGFCdHV19Rv30ksvxcaNG+NTn/rUMddhIR8AJIZiy97q1atj1apVA44vXrw4lixZMujzHThwIL70pS/FX/3VXx36cHAsNH0ASAzF6v2FCxfG/PnzBxxvaGgYcKyxsTF27twZpVIpisVilEql2LVrVzQ2Nh4a85vf/CZee+21uOGGGyLidzMJ5XI53nrrrfjyl798xDo0fQAYYg0NDYdt8Iczfvz4aGpqio6OjmhtbY2Ojo5oamqKcePGHRozadKkeOaZZw79/e677479+/fHbbfdVvHc7ukDQGIkbNlbsWJFrFmzJpqbm2PNmjWxcuXKiIhYtGhRbNq06T1fW8Wkv2fPnvibv/mb6OrqilmzZsW111576GdLliyJu++++z2/MQBweFOnTo21a9cOOH7//fcfdvyxrguo2PSXL18ekydPjssvvzwefvjh+PGPfxx33nlnjBo1KrZt23ZMb5Aq9b4z6N8ZLoXCsS+EyEqx/uSsS6jomqe+k3UJFT102X/PuoSKTh59atYlVPTOnp1Zl1BR75t7si7hqMaMn5R1CRXt2zX4f9fzqGafyLd169a49dZbY+7cufHAAw/EmWeeGX/2Z38W77777nDVBwCcIBWb/oEDBw79ua6uLpYvXx7nn39+3HDDDRo/ALnUVy6f8NdIUbHpn3POOfHTn/6037HbbrstLrzwworPAQaAalXqO/GvkaLiPf2vfe1rUVdXN+D4zTffHFdeeeWQFQUAnHgVm/7YsWOP+LPzzjvvhBcDAFkbSdPxJ5p9+gBQIzyRDwASpRwnfU0fABKm9wGAqifpA0BiJG2xO9EkfQCoEZI+ACTyfE9f0weARJ5X75veB4AaIekDQKIvv0Ff0geAWiHpA0CilOOoL+kDQI2Q9AEgYcseANSIUn57vul9AKgVkj4AJPI8vS/pA0CNkPQBIJHnLXuaPgAkTO8DAFVP0geAhC17AEDVk/QBIJHne/qaPgAk+nK8et/0PgDUCEkfABIW8gEAVU/SB4BEnhfySfoAUCMkfQBIlHKc9DV9AEjYsgcAVD1JHwAStuwBAFVP0geARJ637Gn6AJCwev8EOdj79nC+3aCcdPIpWZdwVFc/+UDWJVT0v5uvz7qEigq972RdQkXv/vaNrEuoanWFkX+3svfNPVmXUNFJo0/NugSGmKQPAImSLXsAQLWT9AEgkeekr+kDQCLPTd/0PgDUCEkfABKSPgBQ9SR9AEjkOelr+gCQyHPTN70PADVC0geAhKQPAFQ9SR8AEpI+AFD1JH0ASOQ56Wv6AJDIc9M3vQ8ANULSB4DEQUkfAKh2kj4AJPJ8T1/TB4BEnpu+6X0AqBGSPgAkSmVJHwCocpI+ACTyfE9f0weARJ6bvul9AKgRkj4AJCR9AKDqSfoAkCj19WVdwpCR9AGgRkj6AJDI8z19TR8AEpo+ADBstmzZEkuXLo29e/fG2LFjo729PaZMmdJvzD333BPr1q2LQqEQJ510Unz+85+Pyy67rOJ5NX0ASBwcAUl/+fLl0dbWFq2trfHYY4/FsmXL4sEHH+w3Zvr06XH99dfH6NGj46WXXorrrrsuNm7cGCeffPIRz2shHwAMsZ6enti+ffuAV09Pz4Cx3d3d0dnZGS0tLRER0dLSEp2dnbF79+5+4y677LIYPXp0RERMmzYtyuVy7N27t2Idkj4AJIbinv7q1atj1apVA44vXrw4lixZ0u9YV1dXTJw4MYrFYkREFIvFmDBhQnR1dcW4ceMOe/5HH3003v/+98dZZ51VsQ5NHwASQ9H0Fy5cGPPnzx9wvKGh4bjP/eyzz8Zdd90VDzzwwFHHavoAMMQaGhqOucE3NjbGzp07o1QqRbFYjFKpFLt27YrGxsYBY3/2s5/FX/zFX8Q3v/nNOPfcc496bvf0ASBR6iuf8NdgjB8/PpqamqKjoyMiIjo6OqKpqWnA1P4LL7wQn//85+Mb3/hGfPjDHz6mc2v6ADDCrFixItasWRPNzc2xZs2aWLlyZURELFq0KDZt2hQREStXrox33nknli1bFq2trdHa2hovv/xyxfOa3geAxEh4OM/UqVNj7dq1A47ff//9h/783e9+d9Dn1fQBIDESmv5Qec9N/4orrojHH398UL9TP+b4VykOlQXfX511CUf1v2Zdl3UJFZX2D9xvOpL09ZWyLqGivgO9WZdQ0UmjT826hIrKVfDNaIVRJ2VdQkWFUfVZl8AQq9j0X3nllSP+bM+ePSe8GADIWrlWk35LS0ucffbZUS4P/A9wtKf+AAAjS8Wmf/bZZ8d3vvOdmDhx4oCfXX755UNWFABkpS/HSb/ilr25c+fGr3/968P+bM6cOUNSEAAwNCom/dtuu+2IP7vjjjtOeDEAkLXD3dLOC1v2ACCR54V8nsgHADVC0geARM0u5AMA8kPSB4BEeeQ/3PE90/QBIJHn1fum9wGgRkj6AJCwkA8AqHqSPgAk8vxwHk0fABJ5bvqm9wGgRkj6AJDos2UPAKh2kj4AJNzTBwCqnqQPAIk8J31NHwASnsgHAFQ9SR8AEr5lDwCoepI+ACTKfVlXMHQ0fQBIWMgHAFQ9SR8AEnnepy/pA0CNkPQBIJHnpK/pA0DCV+sCAFVP0geARJ6n9yV9AKgRkj4AJCR9AKDqSfoAkMjzY3g1fQBI+GpdAKDqSfoAkMjzQr5hbfoLvr96ON9uUB6ZszDrEo7qwP6erEuoaFT96KxLqOik+pOzLqGit/fszLqEqlYYdVLWJRxVXaGYdQkVjfR/Yzh+kj4AJCzkA4AaUe4rZV3CkLGQDwBqhKQPAAlJHwCoepI+ACTynPQ1fQBIlEv5bfqm9wGgRkj6AJDI8/S+pA8ANULSB4CEpA8AVD1JHwASeU76mj4AJPLc9E3vA0CNkPQBICHpAwBVT9IHgERfjpO+pg8ACdP7AEDVk/QBICHpAwBVT9IHgES5lN+kr+kDQML0PgBQ9SR9AEhI+gBA1ZP0ASAh6QMAVU/SB4BEua8v6xKGjKYPAAnT+wDAsNmyZUssWLAgmpubY8GCBbF169YBY0qlUqxcuTJmz54dc+bMibVr1x71vJo+ACTKfaUT/hqs5cuXR1tbWzz55JPR1tYWy5YtGzDm8ccfj9deey3Wr18fjzzySNx9992xffv2iufV9AFgiPX09MT27dsHvHp6egaM7e7ujs7OzmhpaYmIiJaWlujs7Izdu3f3G7du3br45Cc/GYVCIcaNGxezZ8+OJ554omId7ukDQKJvCO7pr169OlatWjXg+OLFi2PJkiX9jnV1dcXEiROjWCxGRESxWIwJEyZEV1dXjBs3rt+4SZMmHfp7Y2Nj7Nixo2Idmj4AJIbiC3cWLlwY8+fPH3C8oaHhhL9XJZo+AAyxhoaGY27wjY2NsXPnziiVSlEsFqNUKsWuXbuisbFxwLjXX389pk+fHhEDk//huKcPAImsF/KNHz8+mpqaoqOjIyIiOjo6oqmpqd/UfkTEvHnzYu3atdHX1xe7d++OH/zgB9Hc3Fzx3Jo+AIwwK1asiDVr1kRzc3OsWbMmVq5cGRERixYtik2bNkVERGtra0yePDnmzp0bV199ddx4441xzjnnVDyv6X0ASIyEh/NMnTr1sPvu77///kN/LhaLhz4MHCtNHwASI6HpDxXT+wBQIyR9AEjkOenXlcvlctZFAABDz/Q+ANQITR8AaoSmDwA1QtMHgBqh6QNAjdD0AaBGaPoAUCM0fQCoEZo+ANQITR8AasSwNP0tW7bEggULorm5ORYsWBBbt24djrfNhT179sSiRYuiubk5rrjiili8eHHs3r0767Kq0qpVq2LatGnxq1/9KutSqs67774by5cvj7lz58YVV1wRX/rSl7Iuqao89dRTcdVVV0Vra2tceeWVsX79+qxLokYNy7P3//iP/zj+6I/+KFpbW+Oxxx6L7373u/Hggw8O9dvmwt69e+Pll1+OGTNmREREe3t7/Pa3v42vfvWrGVdWXX7xi1/E17/+9Xj11Vfjvvvui/PPPz/rkqrKV77ylSgUCnH77bdHXV1dvPHGG3HGGWdkXVZVKJfL8bGPfSweeuihOP/88+Oll16Ka665Jp577rkoFEy2MryG/H9x3d3d0dnZGS0tLRER0dLSEp2dndLqMRo7duyhhh8RcdFFF8Xrr7+eYUXVp7e3N/7yL/8yVqxYkXUpVWnfvn3x6KOPxmc/+9moq6uLiNDwB6lQKMSbb74ZERFvvvlmTJgwQcMnE0P+1bpdXV0xceLEKBaLERFRLBZjwoQJ0dXVFePGjRvqt8+Vvr6+ePjhh2PmzJlZl1JV7rrrrrjyyitj8uTJWZdSlbZt2xZjx46NVatWxTPPPBOnnHJKfPazn42PfvSjWZdWFerq6uLOO++Mz3zmMzFmzJjYt29ffOtb38q6LGqUj5pV5Mtf/nKMGTMmrrvuuqxLqRo/+9nP4sUXX4y2trasS6lapVIptm3bFh/60IfiH//xH+MLX/hCLFmyJN56662sS6sKBw8ejL/7u7+Lb37zm/HUU0/FvffeG5/73Odi3759WZdGDRrypt/Y2Bg7d+6MUqkUEb/7B2TXrl3R2Ng41G+dK+3t7fEf//Efceedd5oWHISf/vSnsXnz5pg1a1bMnDkzduzYEX/6p38aGzduzLq0qtHY2BijRo06dIvuwgsvjNNPPz22bNmScWXV4Ze//GXs2rUrLrnkkoiIuOSSS2L06NGxefPmjCujFg159xg/fnw0NTVFR0dHRER0dHREU1OTqf1B+Nu//dt48cUX45577on6+vqsy6kqN9xwQ2zcuDE2bNgQGzZsiLPOOiv+/u//Pi699NKsS6sa48aNixkzZsSPfvSjiPjdbpzu7u74wAc+kHFl1eGss86KHTt2xKuvvhoREZs3b47u7u54//vfn3Fl1KJhWb2/efPmWLp0afT09ERDQ0O0t7fHueeeO9Rvmwv//u//Hi0tLTFlypQ4+eSTIyJi8uTJcc8992RcWXWaOXOm1fvvwbZt2+KLX/xi7N27N0aNGhWf+9zn4vLLL8+6rKrxve99L+6///5DCyFvuummmD17dsZVUYuGpekDANlzcxgAaoSmDwA1QtMHgBqh6QNAjdD0AaBGaPoAUCM0fQCoEf8PhH6gI+0sCToAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 648x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mf1nUR9XgH4R",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "ff4e4eeb-d522-402f-8cd5-a84b96fbbfac"
      },
      "source": [
        "wandb.init(entity='authors', project='loss-landscape', id='smallcnn_independent_model_investigations')\n",
        "wandb.log({'independent_prediction_disagreement': wandb.Image('independent_prediction_disagreement.png')})\n",
        "wandb.log({'independent_functional_similarity': wandb.Image('independent_functional_similarity.png')})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/loss-landscape\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/loss-landscape/runs/smallcnn_independent_model_investigations\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape/runs/smallcnn_independent_model_investigations</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tjJP7P6E6xxP",
        "colab_type": "text"
      },
      "source": [
        "# Investigating Load Landscape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "88h1ZdcAVyJu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "NUM_EXAMPLES = 1024\n",
        "NUM_CLASSES = 10\n",
        "test_tsne_ds = testloader.unbatch().take(NUM_EXAMPLES).batch(BATCH_SIZE)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4S3MsRn6UJ6P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Given model and test data, return true_labels and predictions.\n",
        "def evaluate_tsne(test_dataloader, model):\n",
        "  true_labels = []\n",
        "  pred_labels = []\n",
        "  \n",
        "  for imgs, labels in iter(test_dataloader):\n",
        "    preds = model.predict(imgs)\n",
        "    \n",
        "    true_labels.extend(labels)\n",
        "    pred_labels.extend(preds) ## change here \n",
        "\n",
        "  return np.array(true_labels), np.array(pred_labels)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-lXOXkNCiEIB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159,
          "referenced_widgets": [
            "4cfdfaa43b754161878c1a19a774e60a",
            "68cee03dfe7b4f01ae9819e588a39501",
            "26b52483f8d84ea5960eddfc2a23e64e",
            "302f3101dd0746aaabb9513846645a13",
            "ef2ed295bda14b3eba3e463017d45d92",
            "4a28c983ba824378a6bcf89d81f0b5f3",
            "edbcbbe69090421c93ce13603a44b60e",
            "f9f9ea7649064c19af12007ac252cce3"
          ]
        },
        "outputId": "415f7cec-d14f-4991-c09c-30d4b94e1d57"
      },
      "source": [
        "NUM_TRAJECTORIES = 3\n",
        "\n",
        "predictions_for_tsne = []\n",
        "\n",
        "for i in tqdm_notebook(range(NUM_TRAJECTORIES)):\n",
        "  subdir = independent_models_tsne[i]\n",
        "  model_files = os.listdir(INDEPENDENT_MODEL_PATH_tsne+subdir)\n",
        "  model_files = sorted_nicely(model_files)\n",
        "  predictions = []\n",
        "  for model_file in model_files:\n",
        "    if model_file=='small_cnn_checkpoint_5.h5': # this check is because this model checkpoint didn't serialize properly.\n",
        "      continue\n",
        "    model = tf.keras.models.load_model(INDEPENDENT_MODEL_PATH_tsne+subdir+'/'+model_file)\n",
        "\n",
        "    _, preds = evaluate_tsne(test_tsne_ds, model)\n",
        "\n",
        "    predictions.append(preds)\n",
        "  \n",
        "  predictions = np.array(predictions)\n",
        "  print(predictions.shape)\n",
        "  predictions_for_tsne.append(predictions)\n",
        "\n",
        "# convert list to array\n",
        "predictions_for_tsne = np.array(predictions_for_tsne)\n",
        "print('[INFO] shape of predictions tensor: ', predictions_for_tsne.shape)\n",
        "\n",
        "# reshape the tensor \n",
        "reshaped_predictions_for_tsne = predictions_for_tsne.reshape([-1, NUM_EXAMPLES*NUM_CLASSES])\n",
        "print('[INFO] shape of reshaped tensor: ', reshaped_predictions_for_tsne.shape)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4cfdfaa43b754161878c1a19a774e60a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=3.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "(39, 1024, 10)\n",
            "(39, 1024, 10)\n",
            "(39, 1024, 10)\n",
            "\n",
            "[INFO] shape of predictions tensor:  (3, 39, 1024, 10)\n",
            "[INFO] shape of reshaped tensor:  (117, 10240)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUEPtIVqPInj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "b935551c-4181-487a-f0bd-cc5cb3dd9cf3"
      },
      "source": [
        "# initialize tsne object\n",
        "tsne = TSNE(n_components=2)\n",
        "# compute tsne\n",
        "prediction_embed = tsne.fit_transform(reshaped_predictions_for_tsne)\n",
        "print('[INFO] Shape of embedded tensor: ', prediction_embed.shape)\n",
        "# reshape\n",
        "trajectory_embed = prediction_embed.reshape([NUM_TRAJECTORIES, -1, 2])\n",
        "print('[INFO] Shape of reshaped tensor: ', trajectory_embed.shape)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[INFO] Shape of embedded tensor:  (117, 2)\n",
            "[INFO] Shape of reshaped tensor:  (3, 39, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zC5vcOlk9fRL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 457
        },
        "outputId": "f1303311-20c7-40e9-c90f-ac33dd029a0a"
      },
      "source": [
        "# Plot\n",
        "plt.figure(constrained_layout=True, figsize=(6,6))\n",
        "\n",
        "colors_list=['r', 'b', 'g']\n",
        "labels_list = ['traj_{}'.format(i) for i in range(NUM_TRAJECTORIES)]\n",
        "for i in range(NUM_TRAJECTORIES):\n",
        "  plt.plot(trajectory_embed[i,:,0],trajectory_embed[i,:,1],color = colors_list[i], alpha = 0.8,linestyle = \"\", marker = \"o\")\n",
        "  plt.plot(trajectory_embed[i,:,0],trajectory_embed[i,:,1],color = colors_list[i], alpha = 0.3,linestyle = \"-\", marker = \"\")\n",
        "  plt.plot(trajectory_embed[i,0,0],trajectory_embed[i,0,1],color = colors_list[i], alpha = 1.0,linestyle = \"\", marker = \"*\")\n",
        "\n",
        "plt.savefig('loss_landscape_2d.png')"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbgAAAG4CAYAAAA3yvKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdaXCk13nY+/+79t6NpbHNglk5FCXNaLUpy7IlS5ZIOjJpS3bJES3l3iTlW/fm061ydP3JyXVSlZIrTlUquY7tip0oDLVEslZGHIkUFcuUbFrWNpQocsjhzAAzWLvRaPT6bufcDy8aA8wAGMwMMA00nl8VS6UB8PbpWfDgnPMshtZaI4QQQvQYs9sLEEIIIXaCBDghhBA9SQKcEEKIniQBTgghRE+SACeEEKInSYATQgjRkyTACSGE6El2txfQUak0UKr3S/IGB7OUy/VuL+Ou2U/vdz+9V5D32+v2wvs1TYP+/syGH981AU4pvS8CHLBv3mfHfnq/++m9grzfXrfX368cUQohhOhJEuCEEEL0JAlwQgghepIEOCGEED1JApwQQoieJAFOCCFET5IAJ4QQoidJgBNCCNGTJMAJIYToSRLghBBC9CQJcEIIIXqSBDghhBA9adc0WxZCCLF7/P1PZ/nM11+kVG1TLCR58P5xzpwodntZt0QCnBBCiDXOXSjx6W+8AgakkzaLDZ/HnzoPsKeCnBxRCiGEWOPscxPYtoFrm0RK49omlmVy9rmJbi/tlkiAE0IIsUap2ibhWERK4wcKL4iwTYNStd3tpd0SOaIUQoh96tyFEmefm7jhnq1YSFJvB1imgWUZRJGmHoT0Z12U1piG0e2lb4ns4IQQYh86d6HE40+dZ7Hhr7lnO3ehxIP3jxOGGj9UOJZBpBRKKU4fH+T8xCLNdtDt5W+JBDghhNiHzj43gWWZ2JaJBhKOtXLPduZEkf/jA2foy7i0vIihQpL3vvUwR0fztP2I81eqXC01UEp3+21sSo4ohRBiHypV26QSFm0/AsC24qSSzj3bW+8b4UgxvfL5QRhxfrJKpBSZlMN8pcVS3WOp6fNXP5zaleUEEuCEEGIfKhaSLDZ8XMfEDxRhpGn5PmnX5vf/898yu9gGrRkZSPMb7zrBmRNFjh/I88rVKn6gODqW49lzUzz9vSs4tkU6Ye26cgI5ohRCiH3owfvHiSJFpDRJ1ySMFJ6vKFVbTJWbaKVQGqbLDf7iqy9y7kKJVMLm2FgeP4iYX2zxk4sLOLaFaRgEoV5zzLkbSIATQoh96MyJIo++99TKPdtIf4q+jINSgAEYJqYBhmHS9sKVoJVNOYyP5Gi0QmYrLdIJi6RrYdtxOFl9zNltckQphBB71EZp/lt15kRxzed/7D99B6U1nSoArcEwNGGk1wSt/lyCIIzIJB2aXkQ25ax8zA8VxULyzt/cNpAdnBBC7EGr0/yTrkWl7q2k+d+uYiGJba0NC5EG0zDIZxyUvpY1Odyf5t1vPkgQRlRqHjPlBlfmapQX27xmvO+217CdJMAJIcQetDrN3wsUWkOkNF/+9kXafnhbz3zw/nGSrgUYaK3igKY1rmvx+mOD/PRShXK1jV4OdO9600HedLJIoxXghwrbssikbL7945k7CrTbRY4ohRBiDypV26ST8bdww4iPE23TYH6xzYuXF3Fsk1zaIZd2yaUdbMu86ZHmmRNF/vE/uI/PffMVZiotQDM6kOY3f+kkx8byTJebTM7Vmas0GR3M0Jd1mVloUsi6WKZJwrWwTAMviFbq6bpJApwQQuxBnTT/hGPhOiaer1DA2ECaQ8MZas2AasNnYckD4mzIp783iWNbpDZJ6e/cy2XzKZ793iQHhjIM96UAyKVdqnWP6XKTyzM15hIW84ttMikbL1D4QUTStXZNookcUQohxB7USfP3ggjTMFBa0WiFNFo+f/jJH/DZb76CUop7DhcYHUzz3E9n0RhoDV6gsK3NJwSkEnYcCGveml8vZBPcO97H+EiWSGlSCZtGO4w7omioNQNmyk2qdZ8//OT3u3pUKQFOCCH2oNVp/s12SMKxAE071Cu9JT/59MtcuFpldCBNsx2SS9m4Tvxt3/Mj0Jr5xdaGr9GfT9Jshzfc6RmGwUA+yWuO9PO+nz1MFCmaXoAXhFTrHmGkKGSdNf0tu0ECnBBC7FFnThT52IffzB/+n28nn3bJpJx4N6duLLouFpIEkca2zLhuzTJoBxHppM1S01/3+f1ZFwyoXLeL6zANg184c4B/9NBrGMgmaLQCTMMgm3ZIJpyuF37LHZwQQvSATm9JL1B4gcKMFJZprOzQHrx/nMefOo9HXIytAcsw+Nn7Rnj16hID+QQHipk1ZQKObTG70OAL37pAy4s2rLV748kh3nhyiH/+x9/GXZ4jF0UKy+zufZzs4IQQogd0dmhJ18KxTdDQ9ELSCZvpcoP7jvSvOdLsy7h85IF7eej+Iwz3p1ioebw0schS49pu7tyFEk/9/RVqrYCka930yHGoL4WGa2ugu4XfsoMTQoge0Nmh+cQ7tEjFO7SfPzPG7EKL2UqLvqzL//Xrp9d0HgE4UIxT/idm67w6tUR/LkH/QIazz03g2CZKxTV2CcfCgw1LAK5fgxdERJHiwfvH78rvwfUkwAkhRA/oBJz16ty8IKJUbbOw1Gax5pNKWBQLKfpzCUwz7suVTjqcGu9jdqHJbKXFuVdKzFVaZNMOQRhPG1A6XHPseStr6AYJcEII0SOu7y3ZkXAsDhYzjA2kqdQ8StUWk3N1psoNBvJJioUkCSeeCjA2mKGQSbDkRyslAJmkjWlqwlDR9EJyKYe5SpOBfJIXLi3cENA+9uE3d+Hd30gCnBBC7BOmaTBYSDJYSFJvBcwvtuL/Ki0KWZfBQpJ82iWdtDl8sI9ffushvvDXr1Jrxg2Yaw2fMNI4psF3fjyDYcC3fjSF61grpQm7aR6cBDghhNiHsimHbMrBDyLKS23K1TbVuk/CtRgqJBkYyPDONx4knbT53DcvUKq2sSyDgbyLaZk8e24K0zTQgFIQhOqmd3R3mwQ4IYTYh67vS/nAzx5mfCRHqdrmynyDRqixtOLM8SLPfO8K2gCD+L5OKVBAtebRn0+ujNcBmQcnhBBim93KbLjOqB3LMtd0PXn0vac4c6JIox0QGiaXJiuUFtvMLLTIpmxM0yRanojqOgb1RoBlGiTda6FkN82DkwAnhBB73HoBa7O7sM6oHcc2iZTGtU18rh0tZpIOQ0M50pZBealNPu1Qawe4toVtmdiWgR9qRgdSeGHcD9O1TfxQdbUs4HpS6C2EEHvc2ecm4vswrQkjhWtv3ki5VG3HtXKRwg8ULS9CKcXMQpNKzSMIIwAc22R0IM0H3nkc0zAII4UfRtSaAZ4f8qvvOHZD8XhnF7gbyA5OCCH2uE6brravUEoThBGWCXOV9evVOqN2XNtEaUUUafxQkU05XJ6pxc9sBIReSDbl8NqjA3z0gXs5+9wEc4stChmXN5wcxLEs0gmH33n4dRQyLsbqy7hdQAKcEELscZ2AZZrx4FPLMmh7EemEzUsTlRuKuld3HHGseGemlObtrx/l+ME8bS/EcW0mSw3KywkjCdfiH/7yqeXsSxvDiI8vS9U2l6ZrOI5JMR+XIKzuZ9lNEuCEEGKP6wQs3fkFBZYJD/zsYbSGybk606uKutfrOPLzp0fJplyuztU5fqDAoYN99CUtWl5EveVTb4VUat6agJdNORwYzBApxWLdZ7rcZGahSX8uQbGQWpk43i3b/ur/8T/+R/7Df/gPfOUrX+HUqVPb/XghhBDX6QSsr/7NZaYXmgwVkjz8jmMrv94p6p5b/q+QcTl+oHBDx5F6K+DS9BLnryyS70tjGAbppE06aTPcD1rrjQOeY5JJ2bT9iIUlj4Ulj3TSZqgvRSHrYnbh+HJbA9xPfvITfvjDH3Lw4MHtfGxX1J8/R+XskwSleZziEP0PPkT29JluL0sIIdbVadP10kQF0zS451DfysdWF3WXqm3KS3FR9/U9KbMph3sO9fHq9BIvXV4gn7AYyF9L+b9ZwKu3ApRa2UfSbIdcnqlh2waDy7vHn16u3LVeldsW4Hzf5w/+4A/4oz/6Iz760Y9u12O7ov78OeYefwzDtjFSKcLqInOPPwaPfkSCnBBiV5tZaPL031+h7YcM9aXWBBDXsThQzDC6WU9K1+KeQwWq7YiJqSpeEDE6kF43gWSrAS8MNbMLLZ57YfautvbatgD37//9v+fhhx/m0KFD2/XIHbXZDq1y9sk4uJkmyvMwE4mVX7+dALf6tWYPjJJ59/skUAohtt25CyWe+JtLREqTTmwcQDbsSbl8fFksJDk1PkCj3mZ2oUXbjzgykltJUtnIzQLe98/PgxEnwgTRzrf22pYA94Mf/IAf//jH/O7v/u5tP2NwMLsdS9mShe99n/KnH8ewbdxCHlVfovzpxykUUgy85c1crpSx02lUu41lmRhozHSKqFJmaCh3R6/lLVRor3qt/eBWf8/2sv30XkHe727zjc+dI5WwUQoMAzIJCy+I+MYPpnjP246t+zVDwLFx8IKIuYUm85UWpXpA49Uyhw/0MTqiuDpXp9QIODXeh2Nbt72+P/3KT+jLJtAaTBMsMy4ar9T9Hfm93ZYA993vfpcLFy7wnve8B4CZmRn+yT/5J/ybf/NveMc73rGlZ5TL9TVntztp8jOfRxkmKE3UbmM6LipSXPrM54nG78HoG8AvzWPaFjoICUtliCLMZJKLz3z7lnZfk5/5PBEGhgbl+TjJJH6jufJavW5oKMf8fK3by7gr9tN7BXm/u9H0fJ100kYbGj9QBKHCsU2m5+tbWnvShIMDSRZrHgHwoxdnsSwD2zSoVlvMl+s02wHf/P7V27pDG8wlWGz4JBwr7mep4i4o/Vn3tn5vTdPYdHO0LcUKv/M7v8Ozzz7LM888wzPPPMPo6Ch//ud/vuXgdrcFpXlwHHQYwnJQNVyXoDwPQP7tb0cHAZHnE1YXQUVgAIkEc48/Rv35c1t+LX9+Lt6SBwFaqRteSwghtkuxkMQPFbZlknTjnVajHZJPO2i9tQ2EaRgM5JO8/kSRk4cK5NIOXhh/73p5cpHHnzpPueatuUM7d6G0pWc/eP84URQHNa31jk/83h3VeHeZUxxCt5Yr/JfPlLXv4wwOoTwPO59n4FfeD74PGGDZmLk8TqGAYdtUzj65pdfRUYSdzaHbHgCGZa15LSGE2E6rA4hhxMeUaM3rjw9yYWpppQXXVmVTDkdH87zuaD8jAym+f34+vofTEClNwrE2bQl2vTMnine1tdeOVOE988wzO/HYbdP3vgeY/cR/wTBNDMdBeR46DOl74EG8iQkM06TvXe9i8emvYw8NodttOvMgtrr7UkFA++KrZN7wRhb/6psQBuhkkqjdRoch/Q8+tNNvUwixz6xXwP3g/eMcLGa5Wqrz0sQi4yM58hn3lp7r2BZjgxlaXkg6YRMpTaQ0tnXr43E2mjq+E/ZlJxPVaGA6DmGlQlSv4YyMMvyhf0hidAzv6hUS4+OYjotTHCKoLIDWmI4DbG33pTyP9qsXUEFA37veBaZB44c/JGrUSIyNMiBZlEKIHbJRAMmkbC7P1Hh1aomh/hRjg+lbLr4e6kux2PB37Xic6+27AFd//hylv/wfYJjYAwNgWWjPQwUB3vQUVj6PMzAIQP+DDzH7ib8Aw0Ank+jlnd5mu6+o1aL96gXQitSJk+goInl4nL5ffBd2obAnLqqFEL0n6drcc6iPqVKD+UqLRivgyGiOhLP1rMhOSzAPduV4nOvtuwC38D+/AkZ8NAlgLR8bLjzxZYZ+/YMkDh1e+dzMfa+l8IvvonHuR/FOb/DGjiara9zs/n7Sr3s9qeMnSJ28BzOZwpucwLAsrNzuTi8WQvQ+0zQ4NJwlm3aYnIuPLOstn2fPTW8pK3KjI9DdMh7nevsuwAXzcxjutfNnrTWGZRFWFnAPHMRc9bGgskBy/AgD73sAM5m64VlrOp4kkwSlEovffIbEoUOYyRRaKcLqIlY+j2Huy3weIcQu1JdNkE7YPPP9K5z9uwkc2yKdsNYUhr9ng7q0u3mHdqf2VYBTvo+VzRE1Gis7OLRGtZo4/YM4g4NrPj8sl7EymXWDG8D85/4H0VI1LjcwLaxsBjOVYvHpp8m96S1EjTo6jLD7+tb9epCel0KI7nAdix+9UsKxreVhpnpNZ5GNCsP3kn21rfCnp8i++S3oKEIHAWhN1Gigo4j+9//qms+N6nVUu419XdDrqD9/jmB6Kq5tMwxQEdFSDR1FK1mW0WIVTBMrl9/wGXOPP0ZYXcTMZFZ6Xt5KnZ0QQtyuUrWNZRoYBljLM9xuNStyN+vpHdza+7EB0vfdR/LIUYof/E0Wn/oa4VIVO1+g732/Rv7Nb1nztcFCGcO2sPv613125eyTYNms9MRZnjSolqokj52IR8cvP3+j48lOz0szkUArhZlIoLj9npdCiN3r3IXSrrq7UlqTTTlUmwHZpI21XBO8m7Mib1XPBrjSV77EwlefgEiBZaHabfzZGYof/A0yr7kPOxu3d7GyWZInTq75Wh2GhIsVnIHBDYNTUJrH7usjXCiDAqy4+LGTZakaDXQQYPcVNlxjUJrHzGRQvo8OQ8xUUrqcCNGDzl0o8fhT57Es86500d+Kydk6bzhZ5NnnpwmVxjT1DVmRuy0o36qePKKsP3+OylefgCiKO5VEEarVBK2o/c3fEFarK5+bODx+wxiIsFIBpbEHN/6DdIpDYBhY2RzYFp3uoe6Bg2RPn4lbfJnGhseTnWeoZhMdhnHRuWFKlxMhetDZ5yawLBPHMuMuIwCGwZN/e7kr65kqNajUPH7udaN89IF71+0s8vc/neXxp86z2PBvqy3XbtCTO7jK2SfRkYqPDSEOcipudhyU54lqcR2aMzyyMgpntaBcwkynsVLrJ5fAco3cf/0LME2ckVEIAnQYUvzgb8bHk4uL2Ln8Snuu9eTe9nOUPv85DMvCyGZXOqpIlxMhekup2iadtFFKx02G0Wg00+UmF65WyaUdcmmXVGJnviWv3onlMw6vPzbIW+8dZmQgzchAemVXFilF248oVVt88msvEimNYWiUZsdH2+yEngxwQWkew7HjRJLVQS4KsfuvJY24Y2M3fG0nuSRxePPCxdTRYxR+8Z00fvJjoqXqmhq5aPl40ipsnD0ZlMvYuRx97/qluM6uUV+3zk4IsfcVC0kWGz6ubcYJHaZBpGCwkCQIFVOlJtDEcUzyaZd82iGbdrC2obxo9fGo65gs1Dy+9aMpDg1nyKYdWl5I249oeyFeoFa+bq7SwrHj063OGLi9loDSkwHOKQ4RhSEEwbUkEK3BssiuSiZZb0JtsFDGsKxNU/sB/JkZUifvYfBXH7nhnq5zPGnn1z+eDBcreJNxc9LksWMMPPQP1tTfCSF6S6cDiE/87ajtR9iWwSPvOMZrjvTjBxG1ZsBS06dS8yhX22BANumQyzjkr9vd3crdWOd41LZMPD/CsSyCSPGlv77EB995Aox4d5ZK2vTnbVKuRdK1KfYlKVfbZFPWyvfKvZaA0pMBrtNiS6VSaM9bTjQx6ful9+AObXy/FSeXLOL0D2x6tBjVakSNOolDh9ZNQomqVaxsDsO+8bfXX6zSvnwZw7LQUYRTHJbgJkSPW90BZLbSIp2wefgdR1d+3XUsBgsWg4UkSmua7ZClhk+t6TNdajJNE8c2yaUdrszX+eJfv4ptW1tKWOkcj4aRjhO+DQPLMml6IacOF0i69g2TuqfLDX7mvlG+/rcXCSONadyYgLIX9GSAy54+Q/sX3kn9+98jXKriDAwy8P6HMTsz4AArfy27sVNO4M/OYGUyDPzqIyQOH97o8fgzMxiOgz1wY41c1GyiPI/E8MiNH6vXqZenMDt3e76POzx8h+9WCLEXdDqAhJHix68uMDqYXvfzTMMgm3LIphwgQxBe291VGz5P/u0EoQJLb+1urHM8mnAsHDv+gdwLIkb6U6STzg2fX617zC60+LnTY4z2JfZ0FmVPBjitFMkjR0keOQpA6uQ9BKUS4dIiZjIZF3Avp++vabflukStJqXP/Q/MRGLdu7Cb794WwTCwC2vLA6Jmk/bFV0kW8zi5HN7EBImDh9bd5QkhepdtmWRSNtW6x+jA+kFuNce2GMhbDOSTaK35hB+Sci0ipfGDiKRrbXo3disNklteyOXZeCr40bE8OdfcUwHtej333bX+/DnKX/4SQWkOO18g93NvJ3n0GOFiBXdsDH9mBmAlfb9TbG04NrrtYaXS6CjasNh6s90bQLi4iJXNrglcqt2OJwxYFmGzydU/+wvC6iLu6AEGHvoVSSoRYp/JZ1ymS02CMMKxt97N3zAMhpdH1ri2iReoldlsG92NbbVBchgpLk0vYZpwbCx3w7HlXtRTAa6zG9NhiJlIEjUaLH7jaXQYkj51L97sLOXPf45wqUriwCH6H3xopdhah0H8EMvCsKx1i61vtntT7VZ8PLnqnk95Hq0Lr4BhELXbXPxvnyIKI6xcnmipytzjj8GjH5EgJ8Q+UlgOcNWGT7GwcTnSejo7MogzMpteiGWw6d3YzRoka625PFPDDxUnDhZuKejuZj1V6H1tN+aAYcTJG6ZB/bt/R7i0xPwn/ztRo4GZSK70fTSSSbTvQxRhWCaGYWxYbH3z3VtcQN4pD1CBvzIbLnn8BItPPxVPL3AciCLMRALDtuO2X0KIfSPp2iQck2rDv+WvPXOiyKPvPUVfxiUMFWnX5pF3HLujo8SpcpNaM+DQUHb57q839NQOrrMbM1SE8nwwDAzDJGo2qH7zmXh3tpzu2un7aBgGURDEASeV2rDY+ma7N4jLA6xMdiWZpX0hnuqdOnESK5UimJvBSiTQSoOO602kNZcQ+1M+41KqtomUuuV6t9U7spcmKiil4x+eb3FCN0Cl5jFfaVEsJBncQyUAW9FTOzinOIT2/biLCXGyiVYRztBIXPztuhidnR1xcFHtFoMPP4KVyaB8D7vQx/A6R4Y3270pz0O1Wlh9BXQU0b74Ksr3SB47Fj878LEy2bj4HMCIf+ulNZcQ+1M+46I11JvBHT1ndCCNFygqNe+Wv7bZDpmYrZFNORwYytzROnajngpw/Q8+hAoCVLsFWi8HE4OBh34l7h0ZBJip1EoCSCe4JMYOMvxbH+bEx/+Iw//8/7khuHV2b+7IyMa7t8UKAHa+QPvSRaJmk+SRo9jLySz+1PKoHqXQQYAGac0lxD6WSTlYlnFbx5SrFbIJUgmL2YUmWustf10QKi7OLGHbJkdGc5i3sfvb7XrqiDJ7+gzhw4+w+PV4FI4zOMTgI7+2ErDmHn8sPpZ03XinF4b0ve8BonrthmGnq222e1upoZuewir0kZ+fxx0aIjF+BHv5Li5qtQgrFZJHjjJwfJyJz32BqNnAGR6V1lxC7FOmYZBPuyw1/Ns+XuwYHcxwcWqJSs1jIH/zY0alNZdmlogixcmDfSv1cb2mpwIcgDtYpPjrH8RMJkmdundlx5U9fQYe/Ug8H648v9L3MXnkKO1XL2Bt0FZrs7u3TtYmlhXX0NWWWPjqExQ/+JtkBwZWgp83dQU7XyD/i+9k/JGHaEYmiSNHcfrXnzUnhNgf8hmXSs2j0Q7vKLmjkHFJJSxmFpr05RI33Y1NzTdotEKOjOZIJ3suDKzoqXcW1pZQ7TYYBonx8RsCUvb0mRt2S96VyXjqdia77jM3272tZG2aJkopDMfBsG1qf/Md7EIhDn4GKyULla9/jfKB+L7NdHonU0kIcXtyaQcMWGr4d5y9uLKLW/I2TRYpV9uUqm2G+lP0526cptJLeiLArTkmzOUo/NJ7yKa3dmEa1Wpx38j1upLcJHMyKM1jLGdeAislCkF5fiX46SgCA6xcDh0EzDzxVXLvfQhD+k8K0TNudzCobZlkUw5LDZ8DxTtL8ihkXNJJm9lKk/78+ru4eivgynydXNrhwAatwnrJnj947RwTBpWF+Jiw2WThyf9J/flzN/1a1W6jPA+7sP7x5M0yJ+3BIqpeB8AwTczluz1ncOha1qZpxjs708RwXdrzpbh8QXZwQvSEzjia2x0Mms+4tP0Iz4/ueC0jA2n8QFFZujGjMggjLs0s4S4nldzJnd9esed3cJ2dEqYZz2DLZNBhuGGrrY7VLb3c0QMM/Mo/WPP5G+3eOrvFoDSP1jqeVmCAkc2tyYqsnH2SsBr3vuzQvk9ioD8OePvgL5cQ+0FnHI1lGgShivs9svXBoIWMy9R8g2rTZ9i9ta4m6z1rvV2c0pqL0zWUghOH8tjWnt/bbMmef5crOyXLwkzE/3uz4unOri+sLmKm0kS1JeYef2zNrm+93dvqr8O2QMVzec10Bt1qrqmh63/wIXQYxkFP65XgN/iOn5fxOEL0kFK1jWubKKUJI03bj7AMtjwYNOFYJF2LpTssF+gYXWcXd2WuTrMdcmQku2NTw3ejPf9OneJQHKgSCTrx+mbF05WzT650NTFsC9NxUcu/nj19ZsPd2+pWYLrdxnAcrEQCO5vj8P/7r9e8xkZZm9mBLLWoN/q8CSGujaOxTYMwiuvQGl4Yt9KK1JZ2S4Wsy2ylteXP30w+4zJdbvCFv75Asx1SyLq8/tgg9792hEK2t5NKrrfnA1z/gw+tW9+2WfF0JzlEex7GckeR1bu+je7eOq3AYDkL0rYAY8Pd4vVZm1pr1OXzmIneaocjxH7WaX6stQFoIgUG8IaTRV68XOHgUPam2Yr5jMvsQotaM7jjzMZzF0o88/0rRFqTdCwWaj7f+tEUx8ZyjA32XreSzez5ALfhTmmT+zenOESw3HlEa4XBtV3fZpmTa3aLy0kiyvO23GpLBwFojeHu7E9Rq+8JneLNfz+EELdv9TiamYUmuZTDB955D6cO9zM5V+fyTI3FusehocyGXfrTCRvbNlhq+Hcc4M4+N4HjWNg6DrYJ2wQDvvZ3k7zh5P5qC7jnAxysX9+2mc6uT0cRWNaa5JDNMidvZ7e4mvbjM3bD3bkMytUDXM1MZmVqgozkEWLndJofV2oel2dqnDxUIJWwuedQgcX8oD0AACAASURBVPnFFtPlJi+2Ag4MZtatUTOWu5pUGz5K6ztqm1WqtkknbZSOMydd28K4hTvBXrLnk0xuR/b0GYYf/QhWNrcmOSR19NimPSc7X2cX+lDNxoaNmTeilgOc6excksnqrFIZySPE3dUp3K7Wl3+YNQyG+9O8ZryflGszOVfnwtUqXnBjSUAh4xJFmkbrzpovFwtJ/FBhmQZJ18Y0DfxQbTgQtZf1xA7udmRPn8HO51HNJun7XgtA6+WXN61763zd7e6EdNDZwe1cgAtK89dm3FlWnBQjI3mEuCtsyySXcqheV7idcC1OHMxTrraZKjd5aaLC2GCGYiG5UjKUS7sYy11Ncunb/x7RuRP0IC5ZCBVRpDYdiNqr9m2AAzAcFxUsxskf9fpN573difrz5yh/8QuoxQXs0QM7di9mDxYJ5ueu7eKQkTxC3E2FjMuV+QZtPyTpXvsWaxgGxb4U+YzL5Hydq/MNFuse48M5Eq6FaRpk03FwPHgH/1xX3wneameVXrOvA5zpOqA0Ogxu2rXkTnTuxbRS2On0jt6L5d/2c5S+8JfLySyujOQR4i4rZOMAV234awJch+tYnDhQYGGpzdVSgxcnKowOphnuS8XBsdGg5YV3VK+2eiDqfrYv7+A6jOW7sHBh4abz3u5E517MXG7ZtVP3YsrzsPsK9P3Su+Mhq+3WLd8TCiHujGNbpJP2yj3cRgbySV4z3hfXrZWanJ9cxF3Ostyuou/9bl/v4Jovn2fhK18iXFrCLhQY/MBvxINRt1mnfk77PiwPJNyJezHvyiSGaZJ9wxtJHj1K5vQbpCWYEF1QyLhMl5sEYbRhaQDEwfDYWJ5KzeNqqc6r00s898IMP3ylTBBGJF2b9/3MIR5+x/G7uPresW93cPXnzzH/2c8QNRqYiQSq3Wb+k/99S02ab5VTHIqDm2HEE7213vZ7sWBhgahWwx0dQ4chZiIpwU2ILsln4tOhrU7r7s8leM14P+demefvXpwjCCNMA7wg4svfucSXn311J5fbs/ZtgKucfTI+MnSXjw3T6R1Lp+/0pdRhGCe0tNvbei+mwxB/6ipWJhNPOPC8NU2ehRB3Vyphc7VU5//7/PN87D99hz/85PdvOl3Atkz+7qfzGLBcB2dgGmBg8PXvXrkr6+41+zbAdZo0m7YTN2s2jB1Lp19TP9duY+fy23ov5k9Po6MI99DhuAG070uAE6KLOu2yqs2AVMLa8gidth9imQaGwfJ/cZBr++FdWnlv2bd3cGubNMd2Mp0+e/oMmfteiz35Cq10P+7IyLY8N6rXCcolnKFhrFSKqNEAkAAnRBedfW4Cx7bQGpSOJwZ43HyETtK18YIIy7x2vaA062Zjipvbt79rd9p263Z0JhEob3ta5mil4sQS18UdHQXiIa4AZvLW5kpJ/0ohtk+p2iaVsPACRRhpbCsuur5Zu6z3/cwhvvydS0QKTCMObhrN+37m0F1aeW/Zt0eUd9p263ZZy1MMtkMwP49qt0kcPIhhxZlaqt0G07ilbilr5uOt6l+5Ewk3QuwHxUKSINI4K3Pi1JbaZT38juM8/PajJByLSGkSjsXDbz8qWZS3ad/u4ODO2m7dLiuZRHlbG2W/GeV5+LMz2IUCdqHv2q+3W7ecQXmtf6WBDvw4qxRuOhVdCLG+1SN0DAOa7RDLZEvtsh5+x3EJaNtkXwe4brBSSXQYoQL/jpoue1evAuAeXHt0odptrGz2lp4VlOYxEol4nA+gHS39K4W4A6vbZc0ttkgnbN7zloPSXeQukwB3l1mp+G5Mtz24zQAXLi4SLVVxDx7EXD6KrD9/joUnv4o/fRWnOMzgw49sefdl9w/Exeiui9YaVIQOI+lfKcQdWN0ua2K2RqXm3dCfUuysfXsH1y3Wcnbj7Saa6CjCu3oFM5Va6bqycoe2WMFMJInqtS3foekwJHP6dFyAbsQTiVVre+v0hNjvxgYzmKbB1flGt5eyr0iAu8tM18WwrJVsx1vlz0yjg4DE4cMr92yre11iGJjJ5JaK1rXWtC9fInH4MMMf/m2cvv64hi6dlv6VQmwjxzYZHUhTawZU69uTZCZuTvbKXWAkEqjbyKSMmk2CUgmnWMRKX5s1tbrXpWEYccPoLdyh+dNTRLUaifFxnIFB8m/9GfzZGfzpaTLLM/KEENujWEhSrra5UmqQS7uYprTS22myg+sCM5m85R2c1jquebNt3NGxNR9b6XWp1JZnwAWVCsHcHE6xiLNqRJCVzQFxAbkQYvsYhsGh4SxBoJitNLu9nH1BdnBdYCaShAsL6DCM0/O3ICyVUM0myaNHb/ia/gcfYvaxT4BSmOn0TWfARa0W3uQEViaLe+Dg2rWlUmCaRPUadl/ful8vhLjRuQulmw4ZzaYc+nMJ5iotBnJJEu7GkwbEnZMdXBeYybg92FaPKVXg489MY+Vy2H39N3w8e/oMgw8/Es+A89qbFq3rMMS7dBHDskgcPXrD/DvDNLGyWdnBCXELzl0o8fhT51ls+KST9qa9Jw8U0xiGwdWSJJzstG3ZwVUqFT72sY8xMTGB67ocOXKEP/iDP2BgYGA7Ht9zOm20lNfGymRu8tngX72K1orEocMbfk7iwCGGf+vDpDe5O+sklajAJ3XinjgpZR1WNoc/dRUVBBt+jhDimrPPTcT330AQKizTQGtj3d6Tjm0xOpBiqtSk2vApZG6/HlZsblt2cIZh8E//6T/la1/7Gl/5ylc4fPgw//bf/tvteHRPMlwXTAPVvvkOLqxWCRcXcUdG1zSGXk1rjWrUV+7PNhLMTMdJJQcPbRpYrWz8MdnFCbE5rTXVusd0uYnSmkjF//mBQinNdLnJ3GILz4/WfF2xL0XCtZiar6OWhyCL7bctO7i+vj7uv//+lf//xje+kU996lPb8eieZBgGppu4aaKJViqueUsmcYaGN/w81Wyiowgrt3EHk3BxEX92FmdwEGdw824KZiqNYVuoeg36bzwSFWI/+vufzvKZr79IqdpmMJ/g7afHKOaTeIEin3ZoBRHJ5Ts1paHtheQzDlPzDaZokHBM8hmXfMYlk3I4OJThme9d4S//6gJLzWDDeztx+7Y9yUQpxac+9Sne/e53b/eje0b9+XOUv/h5gsoCibGDG3bu92dm0L5P8uQ9N9yVrRbVagAb7uBUu5NUkrmhtdd6DMPAysg9nBAd5y6U+PQ3XkETN1CeW2zzl391gQd+dpy3vXaUD7zzOJ98+mX8UOHaJmGkMAz44DtPcO94P7WGT7XpU6q2mV9sY5oGs5UGf/XDq5imSSZ17d4OkCC3TbY9wP2rf/WvSKfT/PZv//Ytfd3g4K31T9yrFr73fcqffhytNU4qhapVKX/6cQqFFANvefPK54XNJkt+DffEYbLHxjZ5IiyVrqIPDFIYu3G3pcKQpRcu4fZnKLzutSutvW6mHY3QnJikkHexNjga3aqhoc2PTnvJfnqvsH/e79Of/REavTxpGzIpB6UUL05W+ej7X8+p40X6+jJ8/n+9wtxCk+GBNB9410neet/auY9RpFhq+CzWPb747YvYtolrWxgYZFM2bT/kGz+Y4j1vO9aNt3mDvf7nu60B7uMf/ziXL1/mT/7kTzA32XGsp1yuo1Tvn0XPfuFLKMPEsC0i38cwNFppLn76L4nG71npKRnMTGPl8wz+2gdo5Tb+aU4rRWMqnuHmz9fWfkxr2hcvEtWWSJ04SbnqAVvL3Fz6wQssfOVLRI0G7sjoml3mrcyOGxrKMX/dunrVfnqvsD/er9Ka0mKLiekl0sn426Vtm5iGgWmYTM/XV34PjhTT/N+/sfbfwUa/P1nHpFb3SCVslI5fJwgVpmGseWY37YU/X9M0Nt0cbVuA+3f/7t/x4x//mD/7sz/DvYVZZPtNe3YOYzmL0rBtiCLQGn9mioWnvkbl61/DsCwM10V5HvOf/iSGZW0YQKJGHZRe93gymJ0lWqrGSSW3MGGg/vw55j/7GVAKI5lcmQ/Hox8BYO7xx+LWYKtmxyGtvUQP0VpTqXlMLzQJAkV/LkEQKRz7Wt3aVua7bWaoL8ViwyfhbN8zxVrbkkX58ssv86d/+qfMzc3xW7/1WzzyyCP8s3/2z7bj0T0nOTK80lLLdF2MZBJME2egyNK3vw1ag2FgWBZWOn3TnpJRrQamcUMAC6tV/Jlp7IEBnKFbmwpQOfskpuNgJhIYq7qjlL/0Rcpf+uLKGnUQYDjOlvpeCrFX1Jo+5ycXmZitY5sGJw7m+fVfPI5S4AVR3FUoiIgitaX5bht58P5xokht6zPFWtuyg7vnnnt46aWXtuNRPe/Arz/Cy3/8ZyjicoFOi63BR36Nuf/+3zCzGVB6ZUL3zeayRfU6VjqzJglFeR7exGXMdHrT2rmNdHpbEkWoKILlOXHBQgl03GqMKEKjIQrBkdlxYu9reSFT5Qa1RoDjmBwZzdGXdTEMgzMnihQK6ZUsyu3IeFw9M267ninWklZdd9nAW97M8KMfie+wyvM4g9fusCrFIcLq4pp6t816SuowRDWba3pT6iiiffFVMAySR49tmn25EWfVOszlQKt9n8RyeUHnY1optO+jGg3svn601rc0SVyI3SAII6bLTRZqHpZpcKCYptiXWkko6XjrfSMcKaa39bVXz4wT208CXBdkT59Z976q/8GHmHv8sTW7u017Si6n8a8+nvQmJ1CeR+r4iS1nTN7qOtZ8bPlrMqdP077wCmGtxuLTT60koFgf+gCM33Nb6xBiu63uFzmYT3D/a0cYyCfRGoYKSUYG0tiWdDDsFfInuYtkT59h+NGPYBf6UM3Gpj0lIb5/MywLMx3/VOnPzsRdTw4cwMrdfnrvZuu4/mNOXz8j/+h/p/AL76Txwk+YfewTBJWFlQSUV//0P29p8KoQO63TL7JS93CduJbti89eZK7S5DVH+jk4lJXg1mNkB7fLbLS7W09Ur2FmshimSbi0hD89jd3fj7tJ15PtWMdGH2v+9IX47lDreLJBIoERBlTOPikZlqLrzj43gWWZWKaBHyhcx0Rrk++fL/HLb5XEjl4kP67sUcr3UZ6HlcsuJ5VcwkylSBzu3j/UcGEBM5vFMAx0GAJgJhKSgCJ2hVK1jWvHAc4woO1HLNbanJ9c5A8/+f11O/+LvU0C3B4V1Zfbc6UztC9dBLjtpJLt4hSH4oxL20ZHEVoplOdtOnhViLulWEjihwrDMFBaU2v6hJHGtsxNx9uIvUsC3B4V1WoYjkMwP49qtUgcObrhtIG7pf/Bh9BhiI4iQMdNoDdJkhHiblpdd1Zr+KABA7Jph4RjYVkmZ5+b6PYyxTaSALdHRfU6OgoJFyu4Y2PYuXy3l7SSgOL09aOaTaJajbDVonL2SUk0EV135kSRR997ir6Mix8oLNMkl3LIpuKZh65tUqpuPuFD7C2SZLLH1J8/x8JXn4i7lOQL5H/hF8m+8U3dXtaKTjLJ7GOfwHAUTi4r7bzErtGpO/vXn/guC3WPbNJZqd2UNlm9RwLcHlJ//lwcKAAzkSRqNqk89XXckdFdFTgqZ5+Ma/C0hjDEsG201iw8+dXbatgsxHZqtkNef3yQb/1oilBpTFPjh0raZPUgOaLcQypnn8SwbQzHWek/aTrOrusDGZTmMVwXw3HiaeOeB1GEPzNF86UXWXj668x+4r8QLJQxUqmVHZ4cY4qdprRmYrbGPYf6+MgD99KXcWm2Q/oyLo++95R0FekxsoPbQ1Z6RBJPIjBME27Sq7IbVrf6slwH7QdxNuVAEcNxWXr22ZXP1Z4XB2zLkno5seNmF5q0/YjjB/LkMy5vukcyfHuZ7OD2EKc4tDKJoFMOsFmvym7pZFMqL549p8MQtGbwkV8jdfw4qtXEzOUwk0kMy0IHAaDxS3PdXbjoac12wGylxUA+QT4jI732Awlwe8jqwNE5+tuNafir23lF9foNLcc69XKGacYNnZf7XVqpNGG12uXVi16klObybB3HNjlQzHR7OeIukSPKPSR7+gxsMIlgt+m081pvKvANzZyjCEyL3P1vo33xVYJKhdp3/46wXJIEFLEtpheaeH7E8YN56Te5j0iA22NupVflbrVRoM687vUsfut/Uf7yFzEcGzMtE8PFnau3AuYrLQYLSfJpOZrcTyTAia7YKFDXv/vd+G7OMNC+h+nE35AkAUXcDqU0k7M1XMfkwDbPchO7n+zVxa4SlOYxUymMZALDMFG+jwZJQBG3ZarcwAsU4yM5rC72aRXdIX/iYle5lilqYiQScZF4u42VzqICv9vLE3tIrelTWmxT7EuutOMS+4sEOLGrXF9igNZgGGTf/GZaL71EVKtt/gAhgEgpJufqJByTA4OSNblfSYATu8p608RHPvq/MfDeBzBsm9arF/BnZ9Bad3upYhebKjXxA8XhkRymaXR7OaJLJMlE7DobJaCk7jmFd2USf3qaqNEkXKqy+NTXpZ+lWGOp6VOuthnqT8nR5D4nOzixZxiWRfLIURIHD9H48Tlm/9t/JagsYGYy0s9SABBGisnZOgnXYmxQsib3OwlwYs9xhoZovvAChmXFd3RRhLmckLLbGk+Lu2uq1CCIFEdGspiGHE3ud3JEKfaksLKAmc1C4KN8HyOKwHF2XeNpcfdUGz4LSx4jAynSSTmaFLKDE3vUSj9LN4HpOOgoQtXr2IX+bi9NdEEYKSbnaqQSFiMDcjQpYhLgxJ7UKSfQvg+2DYYBKiL92tfiTU7E/S3FvnF1vkEYacZHcnI0KVZIgBN70vXlBE7/AMP/6B+T/9n7CRYWaJ1/iahe7/YyxV2wWPeo1DxGB9KkEnLrIq6Rvw1iz9qonMDK5fEmJ2i98jLO8Aju6OjK/DzRW8JIcWWuTiphMdyf6vZyxC4j/+pFz7GyWVKn7sUZHCSYm6X18nlUu9XtZYkdMDlXJ1JyNCnWJwFO9CTDskgcHid57Dg6DGmefwl/fk46oPSQSs2jWvcZHZSjSbE++VsheppdKGCm0/hXJvGvXiVaWiJxeBzTlblge1kQKq7M10knbYb75GhSrE92cKLnmY5D8thxEofHUc0mrfMvElQq3V6WuANX5usopRkfyWLI0aTYgAQ4sW84g4OkTt2LmUjhXb5E+9IldBh2e1niFi0stanWfcaKGZKuHEKJjUmAE/uKmUiQPHkSd+wA4dIizZdeJKwtdXtZYouCMOJqqUEmZTNUSHZ7OWKXkwAn9h3DMHBHRkidPIVhWbQvXMC7egWtVLeXJm5iYraO1prx4ZwcTYqbkgAn9i0rnY7LCYaGCebn4+LwZqPbyxIbKFfb1JoBY4MZEq7V7eWIPUACnNjXDNMkcfAgqRMn0VFE6+WX8WdkoOpu4wfx0WQ27VCUo0mxRRLghACsXI70va/B7uvDn5mm/crLKM/r9rIEoLVmYi5uu3Z4WLImxdZJgBNimWHbJI8cJXn0KMpr03zpRYJyqdvL2vfK1Tb1ZsDBYoaEI0eTYuskwAlxHbuvn9S9r8HKZPAmJ2m9egEVBN1e1r7k+RFT5Qa5tMOgHE2KWyQBToh1mI5L8vgJEgcPEdVrtF56kXBxsdvL2lfio8kahmEwPpLt9nLEHiQBTogNGIaBMzRE+tRrMFyX9qWLtCcuy6y5u2S+2qbRCjlYzODYcjQpbp0EOCFuwkwmSZ28B3d0lLBSofnSizJrboe1/ZDpUoN8xmUgL0eT4vZInxshtsAwTdzRsXjW3MTl5Vlzw7ijYzJrbpucu1Di7HMTzC+2yCQd3nzvEA/dP97tZYk9TP5lCnELrExmedZckWBujtbL54laMmvuTp27UOLxp86z2PBxHYulls+3fjTFTy9LU2xx+yTACXGL4llzh1dmzbVefgl/TmbN3Ymzz01gWSaOZRJGmqRr4zoWZ5+b6PbSxB4mR5RC3Ca7UFguJZjAn7pKzQxQ+SGZNXcbStU2yYSFF0QYBri2ufLrQtwu2cEJcQcM245nzY2PEzWbtF56kWBhodvL2nMG8wka7Xh0UcKxMAwDP1TSlkvcEQlwQmwDZ2CQ/Otfh5lM4U1cpn3posya26JIKc6cKBJF8TQHwwAviIgixYOSZCLugAQ4IbaJtWbWXDWeNbcks+Y2o7Tm4nSNsWKGD737JAO5BM12SF/G5dH3nuLMiWK3lyj2MLmDE2IbdWbNWfl8PDX81Qs4xSLugYNSTnAdrTWXZ2rUmwHjI1kG8kl+7nVj3V6W6CHyL06IHWClUnE5wfAwQalE66UXZdbcda7MN6jWfQ4U01LMLXaEBDghdohhmiQOLM+a03p51ty0TA4HpssNytU2w/0phvvT3V6O6FHbFuAuXrzIhz70IR544AE+9KEPcenSpe16tBB7mpXLkT51L3Z/P/7MDK1XXka192/6+/xii9mFFgP5BAeKmW4vR/SwbQtw/+Jf/As+/OEP87WvfY0Pf/jD/P7v//52PVqIPc+wbZLjR0gePYr2PZrnXyIozXd7WXddpeZxdb5BIetyeFgmBIidtS0Brlwu88ILL/D+978fgPe///288MILLEg9kBBrrJk1d+XK8qw5v9vLuiuWmj4TszUyKZsjIzmZzC123LZkUU5PTzMyMoJlxSMtLMtieHiY6elpBgYGtvSMwcH989Pc0FCu20u4q/bT+93yez0wSHt2jubkJMbMBJmjR3EH+nd2cTtgq++33gq4XGoyMpzjvqMD2NbevP7fT3+XYe+/311TJlAu11Gq93v5DQ3lmJ+vdXsZd81+er+3/F7NFGpknPbEZRa/9zxBuUzte39PWC7hFIfof/AhsqfP7NyC79BW32/bD3nlShXTNDhwqI/Kwt7MJt1Pf5dhb7xf0zQ23Rxty49RY2NjzM7OEi0PgoyiiLm5OcbGpKZFiM10Zs2FS1XK//PLBPNzGMkkYXWRuccfo/78uW4v8Y74QcSFqSUw4MSBAo69N3duYm/alr9tg4OD3HfffTzxxBMAPPHEE9x3331bPp4UYj8zTJPac89hJlOYjoP2fQzHwbBtKmef7PbyblsYKS5MLRFFmhMHCiRcmcot7q5tO6L8l//yX/J7v/d7/PEf/zH5fJ6Pf/zj2/VoIXpeUJrHzGRAa3S7jQ5DDNclKO+tTMvrh5a+6VSR97zlMKnErrkNEfvItv2tO3HiBJ/97Ge363FC7CtOcYiwuoiZSGDYFjoK0UrhDA51e2lb1hlaapoGtmWy1Ap49tw0x8by0lNSdIUciAuxC/Q/+BA6DFGeB5aN9gO059H/4EPdXtqWnX1uAtM0MDBQGjJJG0eGlooukgAnxC6QPX2G4Uc/gl3oQ7db2PkChXe+i8xrX9ftpW3Z/GILNERK49omtmXi2qYMLRVdIwfjQuwS2dNnVsoComaD1vnzBOUy7vBwl1d2c0EYkU7a1FohmaS9UucmQ0tFN8kOTohdyEpnsHI5gvm5Xd+cue2HnL9S5c2nhrCMeAentZahpaLrJMAJsUu5wyPoICCs7N6Wd412wMtXqmitee9bx/nIA/fSl3FlaKnYFeSIUohdysrlMNNpgrk57IHBXde7cbHmceFqFccyOb5c53bmRFECmtg1ZAcnxC7mjoygPI9wcbHbS1ljYanN+ckKCcfi5KE+KeIWu5IEOCF2MStfwEwmCeZmu72UFXOVJhOzdfJpl5OHpP2W2L3kb6YQu5hhGDjDw6hWi3Bpqatr0VpztdRgqtSkL+dyarwfy5RvIWL3kjs4IXY5u6+fpb/5DqUvfRHVbHRl0oDSmsnZOpWaR7GQ5OBQBtPcXXeCQlxPfvwSYpdr/OTHLP6vbxLVlroyaUApzcXpJSo1j7HBNIeGs7su4UWI9cgOTohdrnL2SQzXBaVAKcxEArX86zu1i7u+afIb7ynyC2cOMChF22IPkR2cELtcULo2UcCw459Jd3LSQKdpcqXuYVkGS62Abz8/zdVSfUdeT4idIgFOiF3OyhfQnofpOBjLSR3a93ds0sDZ5yYwDAOtAQyyKUeaJos9SQKcELtY1GyQOX06nhOnFFprlOehw3DHJg3MVVroOLqRcCws05CmyWJPkjs4IXYpHUV4ly+TOnkPifEjLD71dYLyPM7gzmVRLiy1SSdtml5IJmVjLieTSNNksRdJgBNil/KuTKJ8n9SJk1jZLLk3vmlHX29moclMucnPnx7lG9+7QhAqXNvED5U0TRZ7kgQ4IXahoFwmrFRwx8awstkdfS2lNVfm6iwseQzkE5w5OcjYYIazz01QqrYpFpI8eP+49JgUe44EOCF2GdVu4V29gpXL4QyP7MhrrC4DyKYc3nCyyNteN8roQBpAmiaLniBJJkLsIlop2pcuYVgWifEjO1JQvbYMwKTaDHj2+WnmKs1tfy0hukkCnBC7iH/1CqrdJjE+juk4O/Iaa8sAIJuycaUMQPQgOaIUYpcIFysE5TLuyAh2Lr8zrxEpZhaa2LaJZRi4joVpGpiGljIA0XMkwAmxCyjPw5ucxMpkcEZGd+Q1lho+k3N1simHth+RSFgrR6BSBiB6kRxRCtFlWinaly+BAYkjR1e6lWyXMFJcnqnx6tQSlmnw/rcfxTDioKa1xgsiKQMQPUl2cEJ0mT89jWo2SR47hum62/rsat1jcr5OGGlGBlKMDKQxDYOka0kZgOh5EuCE6KKwWiWYn8MZGsIu9G3fcyPF1fkGlZpHKmFxfCxHOnntn7uUAYj9QAKcEF2iAh9vcgIzlcIdO7Btz63UPK7O14mUZnQwzXB/aqXllhD7iQQ4Ie6y+vPnWHjyqwQz01i5PAOP/Nq23LsFoeLKfJ1q3SeVsDgxkiOVkH/iYv+Sv/1C3EX1588x9/hj8P+3d+/BcZ31/cff57JnL5JWd8myZcl2wLFxnAbiEApp3DgmCa0DSS+m7Q9mOpNf+5tfhxI60z+ADgMt7TSZ3mAm9E7LTJte3P6KA0kJiYnjEHJpSoHWWlKokQAAIABJREFUJsbBsS3ZsmRpdVnt9eye8/z+kCXk+0Va7Wr1eWUyE8nS8fd4lf34ec7zPF9merqFfpGxPf+EHYks6PDk+aO2no4EXS1xdd2WFU8BJ7KEJp7+GjgOhCGW62B7UcJi8Zq6c88eszU2VaAtGeUdGzvpakmQiLn0dTcS8/S/tQhom4DIkiqNjf6oK7dz7d25Z4/Zmsz6eBGb0ckC//7KCdK5Im/tbVa4icyjgBNZQpGOTkyxOPPBdXTnfvrVAWx7ZuqxVDZ4EZtY1OWVQyOakhQ5jwJOZAm13vc+TLmEKZcBrrk799hUAcuCIDBEXJtoxCEWcXTMlshFKOBEllDj1ptpufu9uE1JwlwWt7mFrv/14at+/pZsmDlmy3UsIq6NZVk6ZkvkEjRhL7KETBgSXb2Gpl95+zXvfZvMFLlpfTsvfG+I0BiMMeq2LXIZCjiRJRQWCmAMdjx+Td+XyZc4MTzNlvVtrOtpYu93XyUV/T5WNE93QztOcxugk0lE5tMUpcgSCgt5AOx44qq/p+CXOXY6jefarO9JEmlJYVYfpLPTYnVLM4GTZ8+RvRwaO1ypskWWJY3gRJZQmC+AbWNd5aHKpXLIm0NpLAs2rGnGdWz2DRzAtmwswA9LWMysnvz68efY2PYWIval/7c+NHaYfQMHSBXGaY+1sbNvO1s6Ni3GrYnUHAWcyBIKC3nsWOyqlvSHoeHY6TTlIOQta5qJRhwAxvIpHMshwGBhERICcCY/yuHxI9iWQ9Tx5v0bJep4/HDiGP/yxhM4tkPCjTPlp9lzZC+7eWAu5BSAUk8UcCJLIPM//83E01+jOHSKSFs77Q88eNmVk8YYjg9PkyuWWd+TJBGLzFynlCXuxsmWczS4cRzLmenpFvh0xjvpaVhFMShSDHwypSyTxam5a37lzacJTICDQzks41g2oRXy7Inn2dKxiUNjh9lzZO9lA1BkOVHAiVTY3PmTjoMdjRJkMzMfX2Z7wMnRLOmsT29XA80NM9OZ44UJhjLDbOu+hRdPvUI5DLBtGz8sEZqQ+9btoCPeds51gjDAD32KgU+ulCPqeBhjKDPT7BQDw7kRDo+/wVff/DoGg2PNjBSjjkcRn30DBxRwsiwp4EQqbOLpr2G5LpbrEoYhdjSKKZcvef7kyHiO1FSBaeckjx99lbF8iiaviS3tN/K29hu5u+9O1jT2XNVUomM7xO04cTdOV6KTKT9N7GzIGQyFwKc91kYiEmfKn8KzPfzQx8YiYkfw7AipwvhS/DGJLDoFnEiFlcZGsRsaIAznPnep8ycnpoucTuWYYJD9w1/HsWxc22WqOMVLQ/9Bf9NaHNthS8emax5V7ezbzp4jeyni49kRSmEZjOGn1r+XvqZeViW6mSxOEbFdSmGZYugTGkN7tHXBfwYi1aBtAiIVFunoxPg+zC4sMeai509O53wGRqZpiLl8e+plDCEz46yQRCRO1I3yjcEXrruOLR2b2L3xAZq9JLlynmYvye6NP3q+trNvO6EJKZsAz45gjKEUlNjctpFMKXvdv69ItWgEJ1Jhrfe9jzOP/93ZZ16G8Oxhy/PPn8wXyxw5laJIFicWMJYfJepEcSwH13awLRvHmAVPF15u5LelYxO7eeCcqc87Vr+LRq+BY1MnaIu1sirRhWM7C6pBZKko4EQqrHHrzRz+uR08c+oFhhOwKutwT+92Nmy9mSAMGM1NcHDwFMWwwOr2Bhq8JJ3xDjKlLJ4TmbuOH5Zoj7Vd5ndauIsFYGhChrNnSBXGmfYz9DatpjHSUNE6RBaDAk6kwg6NHear/vfItkQIgoBUs8sTxe8wPdBEe7yNU2MZrDDCzWvWsaqpjYgT4b51d888Lwtmnpf5YYkgDNjZt33J67ctm9WNq2iOJjk5PcSxqRO0x9pob1fISW1TwIlU2J/9999iMHMfF4xPoejzL298hZaTd/P29X3cu+0GkokfnW5ysenCam+6bogkeGvrhrnR3KHRgMagRaM5qVkKOJEKa4kmKRRz5M3ZhSYGTNnBBC7jIxFezY5z45pObr7h3MOSr2elZKXNjuaS0SYyTM6N5lY1dGFbWrMmtUU/kSIV1hnvIDi7sOTs3mqwLEyxAT+E8XSRz//Lf1e1xmvVGGlgS+dG2mNtpArjvDHxJtlSrtpliZxDASdSYTv7thM6Fp5vsPIxKDtghZSH14MxJGIurU3Rapd5zWx7ZjS3vrkfg+HNqeMMZYYJTXjlbxZZAgo4kQrb0rGJB4810TUVYlsBZJvw37iFcLIDrJlzJ7tar60/XC1pjDTw1pYNtMVaNZqTmqKAE1kC2979AD/3Qoaf3u9R/sE2IhPNtJXSxF2LIDDLviO3YzusaexhfVKjOakdC15k8tu//du8/PLLeJ5HIpHgt37rt9i6deti1CZSNxq33kzP//0Isae/hj31X7zauImpxlb6O5Pcd3vfBQtMlqtGb2Y0N5w7d99cQ+TqG7yKLJYFB9ydd97JJz/5SSKRCPv37+c3fuM32Ldv32LUJlJXGrfeTOPWm1kL3FPtYipodjTX7CU5mRnizanjWmkpVbHgn7a77rqLSGTmtIVbbrmF4eFhwlDTEiIr3exoTs/mpFosY4y58pddnccee4zDhw/z2GOPLdYlRaQOpIsZjk8M4gclVjV2srqpG9vWaE4q64pTlA8++CBDQ0MX/bWXXnoJx5k5ePWpp57iq1/9Ko8//vh1FZJKZQjDRcvamtXZ2cTo6HS1y1gyK+l+V9K9wrXfb5fVw7B/hiNDA5xwhultXE1iGT2b0+tbe2zbor298ZK/fsWA+/KXv3zF3+TZZ5/lT/7kT/jSl75ER0d9PCwXkcU1+2wu6TVxKnOao1PH6Yi3053o1LM5qYgF/1Tt37+f3//93+eLX/wivb29i1GTiNSxJq9x7tncWD7FDyffJKdnc1IBC15F+YlPfIJIJMJHP/rRuc996UtforVVXYBF5OIuNprrjLfTpdGcLKIFB9wrr7yyGHWIyAo0O5o7nR1hNJ8i7U/T27iGRGT5nuwitUN/VRKRqnJsh96m1axL9hEaw9GpYwxnR3QKiiyYAk5EasLsaK412sJoPsUPJ4+RK+WrXZYsYwo4EakZ80dzgQk0mpMFUcCJSM1p8hrZ2HKDRnOyIOroLSI1aXY01xydOdPy6NQxpgpp/nPku4wXJ2iPtbGzb3vNdT2X2qERnIjUtNnR3Hh+giePPcNYIUXMiTLlp9lzZC+Hxg5Xu0SpUQo4Eal5ju3w3dGDRB0Px3YoheW5/943cKDa5UmNUsCJyLIwlk9hWxYWFhF75umKZ0dIFcarXJnUKgWciNS80IQ0RBrwwxKeHcGxZw5598MS7bG2KlcntUoBJyI1LTQhg9On2NqxGQuLsgkwxlAMfIIwYGff9mqXKDVKqyhFpGYZYziZGSLtT/Ounm2sS/axb+AAqcK4VlHKFSngRKRmncqcZqqYpjvRRXu8jfZ4mwJNrpqmKEWkJg1lhpkoTtIZ76AroT6Tcu0UcCJSc0Zyo3PTkKsauqpdjixTCjgRqSlj+RRncqO0RFvoaeiudjmyjOkZnIhU1aGxw3MLR5JeE5vbNrK14230NvZgWVa1y5NlTCM4EamaQ2OH2XNkL1N+Gs/2mChM8q2h/yBdnFa4yYIp4ESkavYNHMCxHVzLoWxKRF2PqOPxjcEXql2a1AEFnIhUTaowjmdHMIABHMsh6ng6fksWhQJORKqmPdaGH5ZwbQfHmjlEuRAUdfyWLAoFnIhUzc6+7QRhQDHwiVgupbBMseyzY+1PVLs0qQMKOBGpmi0dm9i98QGavST5oEBbtIU71txOe1wjOFk4bRMQkara0rHpnOO3hjLDpArjNHmNNHmNVaxMljuN4ESkpqxq6CLmRDk5PUQpLFe7HFnGNIITkZpiWzZrm9bww8ljfPPkS/zP2OvqHiDXRSM4Eak5MTfGtJ/hmRPPM16YIOHGmfLT7Dmyl0Njh6tdniwTCjgRqUn/MfxfRGwXy7IwGKKOh2M77Bs4UO3SZJlQwIlITUoVxkm4cQD8oIQxBs+OaBO4XDUFnIjUpPZYGyVTxnM8DAY/9CkGvjaBy1VTwIlITZrdBF4Oy7iWS7Hs4wc+O/u2V7s0WSYUcCJSk+ZvAvdDn5ZoM+9e/U46E+3VLk2WCW0TEJGadf4m8MHpU4zkRvEcj5ZocxUrk+VAIzgRWTbWNPaQcBOcnB4iV8pVuxypcQo4EVk2bMumP9lLxI5wPD2IH/jVLklqmAJORJYV13bpT67FAMfTgwRhUO2SpEYp4ERk2Ym5UfqbeikGRQamTxKasNolSQ1SwInIstToNdDbuJpMKcvp7Ei1y5EapFWUIrJstcZaKAZFRvMpoo5HR1xbCORHFHAisqx1J7ooBj6nsyNEbI/maFO1S5IaoYATkWXNsizWNq3hzakTDE6fxHPWET97huXlHBo7zL6BA2rFU8cUcCKy7M1sH1jL0cljHBh8idfHjzBRnLxkcB0aO8yeI3txbOecVjy7eUAhV0cUcCJSFyK2S76U58DJb+HYDg1ugsniFP/8gy/zgfJPsaGln1JYwg9KfPXNrxOYAL/kMxFMEpgQ27LZ+8N/V8DVEa2iFJG68cKpl4m6UVzboRj6GAyBCXj6xDc4mRliJDfKtJ9hyk9jQkOmlKMcBlgGQhMynDujhqp1RCM4Eakbsz3kQhMSmAALC9d1yJfzvLXlBjwngm3ZrEp0cSw9gMXMMzwLC6yZqc59Awc0iqsTGsGJSN1oj7XhhyUc28FzPCJOhBBDZ7yDmBvFtmbe8mZb8WBZGCDEgDE0e01qqFpHFHAiUjdmg6sY+BhjKAY+QRhc0ENuS8cmehq6sbAwxmBbNq2xFhzbUUPVOqKAE5G6Mb+HXK6cp9lLsnvjxVdGfuCG95H0mkh6jXTE2rAt+6JhKMuXnsGJSF05v4fc5b7up/ydPDf4TQrlAh3xdu2FqzMKOBFZsdY09fAzb93F5raN1S5FKkBTlCKyYuVKeRJXceqJLE8KOBFZkUpBiVJYoiGSqHYpUiEKOBFZkXLlPAAJVwFXrxRwIrIiZUs5LCxibrTapUiFLFrAvfrqq2zevJm///u/X6xLiohUTK6cJxGJz23+lvqzKK9sJpPhD//wD7nzzjsX43IiIhUVmpB8WQtM6t2ibBN45JFHeOihh3j++ecX43IiIhVzaOwwXz/+HGfyo3TGO7hv3d3a+1anFjyCO3DgANPT09x3332LUY+ISMXM9oGb9KeIOlEypSx7juxVB4E6dcUR3IMPPsjQ0NBFf+3pp5/mj/7oj/jbv/3bBRfS3t644GssF52dTdUuYUmtpPtdSfcKy+9+Dxx8kWgkggGMMcQjMYrlIgeGX+QnN992xe9fbve7UMv9fq8YcF/+8pcv+Wv/+Z//yejoKD//8z8PwMTEBPv372dycpKPfOQj11RIKpUhDM01fc9y1NnZxOjodLXLWDIr6X5X0r3C8rzf4fQoCTdOMfCxLItyOcA2DsPp0Svey3K834VYDvdr29ZlB0cLega3bds2Xn755bmPP/7xj3PTTTfxoQ99aCGXFRGpiPZYG1N+GtuyCE0IgB+W1EGgTml9rIisGLPtdEphmdAYCuWiOgjUsUU9bPmRRx5ZzMuJiCyqLR2b2M0DfO34NxjLj9Gd6OKe/rsWbRXlobHD7Bs4QKowTnusTd0JqkzdBERkRdnSsYn1zX0cnTpOf3ItSe/yCylmQ2uiNElrpOWSoTW7QtOxHRJunCk/zZ4je9nNxfvRSeVpilJEVhzP8QDwA/+yXzcbWlN+msZIYi60Do0dxhhDOSxTDHxypTxPH38OAMeyCU1I1PFwbId9Awcqfj9ycRrBiciK84PxH/LVo18nU8rQnei66KjMGMMzJ/YDYGFRDPyZUDMBTxz9GiHhOV8/mh8l6kTxwxIWFnHbwbMjpArjS3Zfci4FnIisKLOjssAERJ3o3KjsZ8P76W/uI1/OkyvnyZfzjOTOEHWiBCbAxsGyLKK2R6aUpSvegWM7OJaDYzt0xTuZ8tMAuLYDaIVmtWmKUkRWlH0DB3Bsh6jjYTBYQNkEfPXY1xmYHmQ0P0YQBrREm+mId2BZNjEnSsyNEXU8sKA70Ul3Qxcd8XZaYy0kvSbuXbeDUlimHAY4OBQDXys0q0wBJyIrSqowjmdHsC0bg5l5XmZHyJVybGhex5b2Tby1dQNrGnt437q7MSbED0sYYy4bWhta+nnP6nfS4iXJBwWavSS7N2qBSTVpilJEVpTZzd5Rx8OxZqYdi4FPV6Lzgu7es9sK5lZRepdeRTmcHWVdcz/3rbsb5+wUpVSXAk5EVpSdfdvZc2QvRXw8O3LFqcQtHZvY0rHpskdX5Uq5uQUrCrfaoSlKEVlRtnRsYvfGB2j2kuTK+UWZShzJjeJYDu2x1kWsVBZKIzgRWXFmR2WLIVvKkSllWZXo1uitxmgEJyKyACO5M7iWS3tco7dao4ATEblOGT9LtpSjM9GBbenttNboFRERuU4juVFc26Ut1lLtUuQiFHAiItdh2s+QK+foimv0Vqv0qoiIXIeR3CgRO0KrRm81SwEnInKN0v40+XKeLj17q2l6ZURErtFIdmb01hJtrnYpchkKOBGRazBVnKYQFOhOdGr0VuP06oiIXCVjDGdyZ/BsT6O3ZUABJyJylab8NIWgSHdDJ5ZlVbscuQId1SUicgWHxg7z/P98k5OTp0l6Se7fcC8tnRrB1ToFnIjIZcx2AHcdG8/xKAQF/uWNJ7AsS73eapymKEVELmPfwAFsy8aybGzLJu7GcGyHfQMHql2aXIECTkTkMlKFcRzLJjQhEdvFsiw8O0KqMF7t0uQKFHAiIpfRGm0lHxRxbGeuHY4flmiPtVW5MrkSBZyIyGW8o2srYRhgTIgx5oodwKV2KOBERC4hV8rTFm/lp9a/l7ZE66J1AJeloVWUIiIXYYxhKDuMa7m8e/U7+Zm338Po6HS1y5JroIATEbmIyeIU+XKe3sbVc8/equXQ2GH2DRwgVRinPdbGzr7tGkFeBU1RioicJwgDhrNniLvxqh/JNbsPb8pPk3DjTPlp9hzZy6Gxw1WtaznQCE5E5Dxn8mOUTZn+hrVYlsWhscMcOPgiw+nRJR9B7Rs4gGM7eHYEgyHqeBTx2TdwQKO4K9AITkRknkK5yFg+RWu0hUQkPjeCmsxPVWUElSqM41oOhaCIH5QAtA/vKingRETmOZ0dxrZsuhu6gB+dZOLYztwIailPMmn2kmTLOQA8JwJoH97VUsCJiJw1VZwmU8rSnegkYrsEYcCZ3CihCSmWfQpBkWLg42Azlk9VvJ6R3Cib2t5KGIYAWFjah3cN9AxORAQITcjp7DBRJ0prtIUzuTFG8ykSkQSFcoEGN45fLlEOA4pBkYZIA2dyo7TGWonYV34rvZaVkKEJOZU5zWRxir5kLw/Gd/Hy6de0ivIaKeBEZEWbDZ4zuVESkQQ/3nMbQRhQNmWavEbet24nTxz9d8phGddyCQixLZvbV93KSG6UM7kxktEm2mNtNEQSFw0ygD1H9s4c1uzE5p7j7ebCDePlsMyJ9ElyZ6clVzesoj3exjt73rHkfzbLnQJORFas2QUktmXj2i65Uo5nTuznvX0/yY+v3kYikmBdso+o43Fg+MJVlIVykfHCBBPFKaaKaU5nRnjh5EtEnAgJN85kcYp/+sG/4doRAhOABYEJLrkSslAucDw9SCmcWUzSneikPa5nbddLASciK9bsEnzbsgjDkKjjEWI4mHqdu/vvnPu6LR2b+MnNt11wkknMjbK6cRWrGrqYLE7x5JvPYM7+44c+BkNoQsbyKdpiLVhYc5vGz18JOe1nGJg+RWgCANpjbXQlOpfgT6F+KeBEZMVKFcZJuHEAbMfBsWyMMde8BN+2bNpireTLeRoiCQITEBqDYzlEXJfpUpayKdPgJLCtmbV981dCpvITDGVPz12vJdpMT0P3It3lyqVVlCKyYrXH2vDDEpZl4VwkeK7nemUT4DkeMTeK50Qohj7NkSYsLMomOKcjwd1r72QoM3xOuDV5jaxp7MGyrEW5x5VMASciK9bOvu0EYUAx8BelFc7518uV8vhBiXvW3cUv3vizNHvJuY4EP/fW99MYbSBVGCfqeAAk3AR9Tb1zozxZGE1RisiKtaVjE7t5YNEOMp5/vdH8GHE3zvbe93DHmndhWdbcdf2gxIn0ANN+hpZoC1PFKWJOlHXJtQq3RaSAE5EVbUvHpkXdU7alYxMbWvo5OnmcmBtjQ3P/OdONuVKeE+lBQkJ6GroZyY0RsSOsa+6/oGuBuggsjP6qICKyiEpBiePpQVzbpf+86cbJ4hRvTh3Hsiz6mnoZzaewLYv1zX0XbBZXF4GFU8CJiCySIAw4nh4gNCH9ybVEzp4dCXAmN8rg9Cnibpx1ybUMZYYJjWF9sg/v7DO4+eZ3EQhNuORnYNYDTVGKiCzA7DTiWD5FQ6SBrR2bubP33cTdGHDusVvN0SQ9Das4PnWCUlhiQ3M/sbNfd75UYZyYE6UQFAGIWba6CFwjjeBERK7T/GnEiB1h2p/m5dOvMZA+Ccwcu3VsaoDJ4hRdiU56G1czOH2SYuDTn1xLIpK45LXP7yJgWZa6CFwjBZyIyHWanUZ0LJuAgLgbw3M89g0coFAu8sPJY+TLedY2raEr3sHA9EmypRy9Tatp8hoved0zubFzugjY2OoicB00RSkicp1ShXHiToxi6ONYDu7ZhSKj+TGOTh3HxmJDcz9xN87JzBDTfobVDatoiTZf9Hoz05nDTBYn6U+uZVWiixeHXtUqyuukgBMRuU7tsTZShXEsyyJiu1iWRa6UJ+7Gidgu65J9eE6Eocwwk8Wpyx6ePL+LgIXF+uZ+GiIJbl11yxLfVf3QFKWIyHXasfYnKAWlmalEA9lSjmLg8+M927iheR2eE+FMbnRuBHapw5NnpzPPDzdZGI3gRESuU09jN3eseReHx3/IWCFFY6SBnX0/yU+cPbkklZ9gJDd62cOT53cRuJZw0ybwK1PAiYhch5k2OOPc0LKeja03UAiKc81JYWZT91D29GUPTx7Lj3M6OwzMdCRYl+y76nC72gaqK9miBNzf/d3f8fjjjxOJRLBtmyeeeGIxLisiUnPmOoDnx0i4cW7pvIl1zX2sS/bNrYyc9jOcnB665OHJxhiGssOMFyaAqw+30IRkSzmeOvbsXAPVsilfsoHqSrfggHvmmWd4+umn+dd//VcaGxsZGxtbjLpERGrO7MjJsR0iZzuAv3jqVVY39MyFW66U40R6kKjjXfTw5CAMGJg+SaaUBS4fbqEJyZcLZEpZMn6WfDmPwTBeGCfqRLGx51ZuahP4hRYccH/zN3/Dww8/TGPjzIvb0dGx4KJERGrR7L63iO1SCIpzR2y9cOol3t69lUK5wLH04CUPTy4GPifSAxQDH7gw3IwxFIIiGT9LppQlW8piMADEnBhRx6MQFGnymigGRaKONzf1qU3gF1pwwB09epTvfe97fP7zn8f3fX7hF36B3bt3L0ZtIiI1ZX4HcM/25pqkpgrj+IHPsfTAJQ9P/vbwd/n6if2k/TRNXhNv79zK9rXvwbVdUvkJMqUM2VJuZuoRiDoerbEWGiMNxN0YZ3JjTBQniTpR3rduJ08c/Xf8sIRnR/DDkjaBX8QVA+7BBx9kaGjoor/20ksvEQQBp0+f5h/+4R+YmJjgF3/xF1m/fj233XbbNRXS3n7pXf31prOzqdolLKmVdL8r6V5h5d3vqmQnk/kpom6U2WOUi+UiXU3tTFijJJMxbuy4gXjk3PMlDxx7mf939Elc2ybhxSmEBb41/CpNTXH6mteAA54XoT3aTTLaSFO0Ee/sQc3pwjTHJ08Sxsrc2NFPT1MXtmXT1d7MVw4/y5lsiq7Gdt6/6b0A/OnBv575XMPM597ec9N13+9yf30tY4xZyAV27drFpz/96blA+8xnPsPatWt56KGHruk6qVSGMFxQKctCZ2cTo6PT1S5jyayk+11J9wor836ff/21uWdwsyOnclDmjjXvYnXjKjY0959zvqQxhpHcGb548HHy5QIR2yE8O+VYDgOSkUb+99YP0+g1znX1nhWEAcO5M4wXJjidGeHg2OtM+lOX3BIw//ng/FHd7o3Xt7JyOby+tm1ddnC04CnKXbt28c1vfpPbbruNXC7Ht7/9bd773vcu9LIiIjXn/A7gbdFWburZTHdD5wWHJwdhwMnMEGl/mmwpS1OkkYAQB2amNm3IlnMXPdkk42c5mRmiFJaYLEzx4qlXcGznnC0Bt6e38cbk0bl9cBl/5nldaMK59jorfWXlggPul3/5l/nUpz7FT//0TwPwgQ98gPe85z0LLkxEpBbNdgA3xnBiepBpP8PapjXnHJ7sByVOpAcoBEV6GlbRnehiyk+fM0orhv4Fi0Lmj9o826OvqZenjz1HSIiFNXfmZTHwefrEN+iItxFzoowVxpkoTM49r7POPhtc6SsrFxxwsViMP/iDP1iMWkREatr83m+NkUZu6tjEu3q2nXN4cq6U50R6kJBwbm/czr7t7DmylyL+JReFZEpZTk7PjNpc2yXiuAxMnyRVSBF3Yzi2izEhgQnIBwVCE2KMoWh8bMvCtmxy5Twt0aRWVp6lk0xERK7Cd04fnDs9JGJHSPtpXh56jXXJvnNOLzk5PYRru9yQXDfXzPT8qc2oHcV1XP75yJdpO9HKrd0/RkvsRyFZDstYWHTGO+hOdDFdysytyjTGEIQBjnXuFoQmt4Gp0rRWVs6jw5ZFRK7CVw4/i23ZhIQEBMTcGJ470/sNYCQ3yuD0KeJunLe0rL+gU/eWjk08/I7/wwc3Pogf+mdPIIkyVkjx1LFnGUifxMKiJdrM+mQ/N7a+hVUNXdzTfxdBGFAMfIwx+GEJ13YvHd5qAAAJp0lEQVTm9s5ZzExFRiNRehLdNHtJcuU8zV7yuheY1AuN4ERErsJIZgwMM4kCcyOqsXyKgemTTBXTtESbWdPYc8HpJfOdv1l8ZjO4xevjR7hv3d0XbA4/f/TXHmvj5vabeHX4NfywRMKNEZiQIAz42Y33r+hAO58CTkTkKiQiMdL5DIEJyQcFxsNJbMum2UsyVUzTneiiK3Hlk5zO3SwewbZsLMci7U9fEG6zZhe2AEwUJjmVOU08EuX7qR8wUZxUN4FLUMCJiFzB0cnjvLP3Fp46/BzFoMjsMC40AbkgT6FUuKpwg5kmqbMrKl1r5i24GFy4onLW/IUtTV4TW9pvZHP7jezs286963Ysyv3VKwWciMhFzAbLSG6UhkiCd697B63RFkYLKYIwwLZtGtw4nuPx0unXuK3nHVd13atZUTm/htmFLa7tMlWc4qWh/6C/ae0lR3vyIwo4EZHzzAYLMNc14MCxVymUiqxKdGIwlMNg7jnctew1u9gztUtNL+4bODD3PM8QkojECUzINwZf4KbOzYtwp/VNAScicp7ZYAkJAWj0GjBWSMbP4ocloo6H58wEz+WmFy9l/jO1yxnNj+FYDpZlzRzubDs4xqzozdvXQtsERETOkyqM49kRXMslanvYlo3neDiWc86S/WLgV2SvWWhChjLDxN2ZEVvUic5NSa70zdvXQiM4EZHzzF8IMssPfFY3rGJn3/arml68XqWgxMD0KXLlHD+x5sfZd+J5SlexeXv2mWGl6lqOFHAiIue52EIQYxl2rt9+1dOL1yNTyjKYPkVgAtY2rWFrx9toj7VeMbjmdxJIuPG5A5l3o43eIiIyz8UWgvzszffR6/ZX5PczxjCaTzGSO0PU8Vjf1HfOMV9XCqlnTzyPZVlYZ7cvqJPADAWciMhFnB8sleqPNr+tTrOXZE1jzxW3APhBiWwpy1g+RSEoMpwbIepEKVPGNhau5a74TgKggBMRqZp8ucCJ9CClsERPwypGsmd47Lt/fc505I1tbyFfLpAr55koTFIKSxdcp8lrIl/OE3fiuFqMMkcBJyKyROYvBEl6TbTH2jmdHSZXzhF34kyXMsTcKJ4dYayQ4u9f38Mda95FX7L3gmsl3AQd8XaavAZsbPYc2UtgAhxjq5PAWQo4EZElcM6pJJbLSG6U4+lBGiMNJNw4I/lRjAmxLAvjzJxRaVs23x09SF+yl8ZII83RJhojjXhO5JxrX8vm8ZVEAScisgT2DRzAsqy5zeN+WMLCIh8UiLoexsy0KsiV83hOBAcb142RL+e5qX3zXBPTS6nk6s7lSgEnIlIh86ckJwtpEpE4UXdmb10YhlhYhOFM4Nm2TRiGhGFI3IlhWRbFwKcz3nHFcJOL00kmIiIVMDslOVmcmlm+b0G2lCUMQxJunIjjggWu7RJzorR4SSyLuRWUlTolZSVRwImIVMBsY1MALOZ6wE2XshhjiDkxjDEkInEsrLObtBN0xTvUkXuRaIpSRKQCZhubhiYkJCQRixO1PSZLaXLlPN2JTu5Y/S7emDz6o83kb1FH7sWkgBMRqYD551k6zIzkHMdhQ7yfh9/xf+Z95c7qFLgCaIpSRKQCdvZtX5LOA3JpCjgRkQrY0rGJ3RsfoNlL6plalWiKUkSkQrQ3rbo0ghMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbqkgBMRkbpUM4ct27ZV7RKWzEq6V1hZ97uS7hV0v/Wu1u/3SvVZxhizRLWIiIgsGU1RiohIXVLAiYhIXVLAiYhIXVLAiYhIXVLAiYhIXVLAiYhIXVLAiYhIXVLAiYhIXVLAiYhIXVLALZFjx47xwQ9+kHvvvZcPfvCDHD9+vNolVczExAS/8iu/wr333sv999/PRz7yEcbHx6tdVsU99thj3HjjjRw5cqTapVRUsVjk05/+NPfccw/3338/n/rUp6pdUkXt37+fBx54gA984AO8//3v55lnnql2SYvq0UcfZceOHRf87NbFe5aRJfHhD3/Y7N271xhjzN69e82HP/zhKldUORMTE+aVV16Z+/iRRx4xn/jEJ6pYUeUdPHjQPPTQQ+auu+4yP/jBD6pdTkV99rOfNb/3e79nwjA0xhgzOjpa5YoqJwxDs23btrnX9PXXXze33HKLCYKgypUtntdee80MDQ1d8LNbD+9ZGsEtgVQqxfe//3127doFwK5du/j+979ft6OalpYWbr/99rmPb7nlFoaGhqpYUWX5vs/v/M7v8JnPfKbapVRcNptl7969PPzww1jWzEG3HR0dVa6qsmzbZnp6GoDp6Wm6urqw7fp569y2bRs9PT3nfK5e3rNqpptAPTt9+jTd3d04jgOA4zh0dXVx+vRp2traqlxdZYVhyD/+4z+yY8eOapdSMZ///Od5//vfT29vb7VLqbjBwUFaWlp47LHHePXVV2loaODhhx9m27Zt1S6tIizL4nOf+xy/9mu/RiKRIJvN8pd/+ZfVLqvi6uU9q37+GiI16bOf/SyJRIIPfehD1S6lIr7zne9w8OBBfumXfqnapSyJIAgYHBzkbW97G//2b//Gb/7mb/Lrv/7rZDKZapdWEeVymb/4i7/gT//0T9m/fz9/9md/xsc+9jGy2Wy1S5OroIBbAj09PYyMjBAEATDzJnHmzJkLpgXqzaOPPsqJEyf43Oc+V1dTOvO99tprHD16lLvvvpsdO3YwPDzMQw89xIsvvljt0iqip6cH13Xnpq5+7Md+jNbWVo4dO1blyirj9ddf58yZM9x6660A3HrrrcTjcY4ePVrlyiqrXt6z6vNdp8a0t7ezefNmnnzySQCefPJJNm/evKyG+tfqj//4jzl48CBf+MIX8Dyv2uVUzK/+6q/y4osv8txzz/Hcc8+xatUqvvjFL3LHHXdUu7SKaGtr4/bbb+db3/oWMLPSLpVK0d/fX+XKKmPVqlUMDw/z5ptvAnD06FFSqRR9fX1Vrqyy6uU9Sw1Pl8jRo0f5+Mc/TjqdJplM8uijj7Jhw4Zql1URb7zxBrt27WLdunXEYjEAent7+cIXvlDlyipvx44d/Pmf/zkbN26sdikVMzg4yCc/+UkmJydxXZePfexjbN++vdplVcxXvvIV/uqv/mpuUc1HP/pRdu7cWeWqFs/v/u7v8swzzzA2NkZraystLS089dRTdfGepYATEZG6pClKERGpSwo4ERGpSwo4ERGpSwo4ERGpSwo4ERGpSwo4ERGpSwo4ERGpSwo4ERGpS/8fFUoPEExPP3kAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s21FDW4qBbA1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "611c87ef-9f04-4421-be94-a9dd2466f0d5"
      },
      "source": [
        "wandb.init(entity='authors', project='loss-landscape', id='small_cnn_trajectory_investigation')\n",
        "wandb.log({'loss_landscape_2d': wandb.Image('loss_landscape_2d.png')})"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/loss-landscape\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/loss-landscape/runs/small_cnn_trajectory_investigation\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape/runs/small_cnn_trajectory_investigation</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}