ex.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/history-lab/foiarchive-api-python-example/blob/main/ex.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "ouObdRkP__QE"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "jubSUYU3CJUR"
      },
      "outputs": [],
      "source": [
        "def call_api(api_url):\n",
        "    \"\"\"Call api_url and return results in a dataframe.\"\"\"\n",
        "    response = requests.get(api_url)\n",
        "    if response.status_code == 200:\n",
        "        return pd.DataFrame(response.json())\n",
        "    else:\n",
        "        print(f'{response.status_code} returned by {api_url}')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8GgJCO5NCu0c"
      },
      "source": [
        "Queries expressed as API Calls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "cMup-Ho-CpLJ"
      },
      "outputs": [],
      "source": [
        "pdb_cmc = (\"https://api.foiarchive.org/docs?\"\n",
        "           \"select=doc_id,corpus,classification,authored,title,body,source&\"\n",
        "           \"corpus=eq.briefing&authored=gte.1962-10-16&authored=lt.1962-10-28\")\n",
        "# Same date range, fewer data attributes, but reference Kruschev in text\n",
        "pdb_ck = (\"https://api.foiarchive.org/docs?\"\n",
        "           \"select=doc_id,authored,title,source&\"\n",
        "           \"corpus=eq.briefing&authored=gte.1962-10-16&authored=lt.1962-10-28&\"\n",
        "           \"full_text=wfts.khrushchev\")\n",
        "# Multi-collection query of documents mentioning hussein and rumsfeld\n",
        "drsh = (\"https://api.foiarchive.org/docs?\"\n",
        "        \"select=doc_id,corpus,authored,classification,title,body&\"\n",
        "        \"classification=in.(secret,top%20secret)&\"\n",
        "        \"full_text=wfts.hussein%20rumsfeld\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q57znUyjEUr1"
      },
      "source": [
        "Call the FOIArchive API and show the results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "3cctPwcUDaVL"
      },
      "outputs": [],
      "source": [
        "df = call_api(drsh)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 678
        },
        "id": "pt4i5LacDzJq",
        "outputId": "952906a3-c12d-4c8f-f2a6-b9c58e29e3cf"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>doc_id</th>\n",
              "      <th>corpus</th>\n",
              "      <th>authored</th>\n",
              "      <th>classification</th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>frus1969-76v26d275</td>\n",
              "      <td>frus</td>\n",
              "      <td>1976-03-27T17:45:00+00:00</td>\n",
              "      <td>secret</td>\n",
              "      <td>275. Memorandum of Conversation</td>\n",
              "      <td>\\nPARTICIPANTS\\nThe Secretary\\nAssistant Secr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>frus1969-76v26d292</td>\n",
              "      <td>frus</td>\n",
              "      <td>None</td>\n",
              "      <td>secret</td>\n",
              "      <td>292. Memorandum of Conversation</td>\n",
              "      <td>\\n\\nPARTICIPANTS\\nSecretary Kissinger\\nAmbassa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>frus1969-76v26d255</td>\n",
              "      <td>frus</td>\n",
              "      <td>None</td>\n",
              "      <td>secret</td>\n",
              "      <td>255. Memorandum of Conversation</td>\n",
              "      <td>\\n\\nPARTICIPANTS\\nPresident Ford\\nDr. Henry A....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>frus1969-76v18d93</td>\n",
              "      <td>frus</td>\n",
              "      <td>1974-11-26T00:00:00+00:00</td>\n",
              "      <td>secret</td>\n",
              "      <td>93. Memorandum of Conversation</td>\n",
              "      <td>\\nPARTICIPANTS\\n\\nTeng Hsiao-p'ing, Vice Prem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>frus1969-76v26d284</td>\n",
              "      <td>frus</td>\n",
              "      <td>None</td>\n",
              "      <td>top secret</td>\n",
              "      <td>284. Minutes of National Security Council Meeting</td>\n",
              "      <td>\\nPrincipals\\nThe Vice President\\nSecretary o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1973NATO05958</td>\n",
              "      <td>cfpf</td>\n",
              "      <td>1973-12-06T00:00:00+00:00</td>\n",
              "      <td>secret</td>\n",
              "      <td>REPORT TO NATO MINISTERS ON THE SITUATION IN T...</td>\n",
              "      <td>FOLLOWING IS TEXT OF C-M(73)117 (REVISED), THE...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1973STATE199684</td>\n",
              "      <td>cfpf</td>\n",
              "      <td>1973-10-07T00:00:00+00:00</td>\n",
              "      <td>secret</td>\n",
              "      <td>n/a</td>\n",
              "      <td>1.  FRENCH NATO DRAFT RECEIVES FAVORABLE REACT...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>CIA-RDP79M00095A000300010001-0</td>\n",
              "      <td>cia</td>\n",
              "      <td>1977-06-07T00:00:00+00:00</td>\n",
              "      <td>secret</td>\n",
              "      <td>FRM/NSC-11 SUBCOMMITTEE REPORTS TO THE SCC</td>\n",
              "      <td>Sanltlzed Copy Approved for Release|201 1/01/2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>CIA-RDP86M00886R000400020064-9</td>\n",
              "      <td>cia</td>\n",
              "      <td>1984-02-06T00:00:00+00:00</td>\n",
              "      <td>secret</td>\n",
              "      <td>IESG MEETING:  SUMMARY OF DISCUSSION</td>\n",
              "      <td>\\n\\nDeclassified in Part - Sanitized Eopy App...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>CIA-RDP86B00420R000200350001-1</td>\n",
              "      <td>cia</td>\n",
              "      <td>1984-02-24T00:00:00+00:00</td>\n",
              "      <td>top secret</td>\n",
              "      <td>NSPG MEETING ON LEBANON 24 FEBRUARY 1984</td>\n",
              "      <td>Approved For Release 2009/10/06 : CIA-RDP86B00...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>CIA-RDP86M00886R002700030022-9</td>\n",
              "      <td>cia</td>\n",
              "      <td>1984-01-27T00:00:00+00:00</td>\n",
              "      <td>top secret</td>\n",
              "      <td>DD/S&amp;T WEEKLY ACTIVITIES FOR 27 JANUARY 1984</td>\n",
              "      <td>\\n\\n25X1\\nApproved For Release 2008/10/16 : C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>CIA-RDP85T00287R001300450001-0</td>\n",
              "      <td>cia</td>\n",
              "      <td>1984-01-31T00:00:00+00:00</td>\n",
              "      <td>top secret</td>\n",
              "      <td>UNTITLED</td>\n",
              "      <td>\\n\\n25X1\\nSanitized Copy Approved for Release...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                            doc_id corpus                   authored  \\\n",
              "0               frus1969-76v26d275   frus  1976-03-27T17:45:00+00:00   \n",
              "1               frus1969-76v26d292   frus                       None   \n",
              "2               frus1969-76v26d255   frus                       None   \n",
              "3                frus1969-76v18d93   frus  1974-11-26T00:00:00+00:00   \n",
              "4               frus1969-76v26d284   frus                       None   \n",
              "5                    1973NATO05958   cfpf  1973-12-06T00:00:00+00:00   \n",
              "6                  1973STATE199684   cfpf  1973-10-07T00:00:00+00:00   \n",
              "7   CIA-RDP79M00095A000300010001-0    cia  1977-06-07T00:00:00+00:00   \n",
              "8   CIA-RDP86M00886R000400020064-9    cia  1984-02-06T00:00:00+00:00   \n",
              "9   CIA-RDP86B00420R000200350001-1    cia  1984-02-24T00:00:00+00:00   \n",
              "10  CIA-RDP86M00886R002700030022-9    cia  1984-01-27T00:00:00+00:00   \n",
              "11  CIA-RDP85T00287R001300450001-0    cia  1984-01-31T00:00:00+00:00   \n",
              "\n",
              "   classification                                              title  \\\n",
              "0          secret                    275. Memorandum of Conversation   \n",
              "1          secret                    292. Memorandum of Conversation   \n",
              "2          secret                    255. Memorandum of Conversation   \n",
              "3          secret                     93. Memorandum of Conversation   \n",
              "4      top secret  284. Minutes of National Security Council Meeting   \n",
              "5          secret  REPORT TO NATO MINISTERS ON THE SITUATION IN T...   \n",
              "6          secret                                                n/a   \n",
              "7          secret         FRM/NSC-11 SUBCOMMITTEE REPORTS TO THE SCC   \n",
              "8          secret               IESG MEETING:  SUMMARY OF DISCUSSION   \n",
              "9      top secret           NSPG MEETING ON LEBANON 24 FEBRUARY 1984   \n",
              "10     top secret       DD/S&T WEEKLY ACTIVITIES FOR 27 JANUARY 1984   \n",
              "11     top secret                                           UNTITLED   \n",
              "\n",
              "                                                 body  \n",
              "0    \\nPARTICIPANTS\\nThe Secretary\\nAssistant Secr...  \n",
              "1   \\n\\nPARTICIPANTS\\nSecretary Kissinger\\nAmbassa...  \n",
              "2   \\n\\nPARTICIPANTS\\nPresident Ford\\nDr. Henry A....  \n",
              "3    \\nPARTICIPANTS\\n\\nTeng Hsiao-p'ing, Vice Prem...  \n",
              "4    \\nPrincipals\\nThe Vice President\\nSecretary o...  \n",
              "5   FOLLOWING IS TEXT OF C-M(73)117 (REVISED), THE...  \n",
              "6   1.  FRENCH NATO DRAFT RECEIVES FAVORABLE REACT...  \n",
              "7   Sanltlzed Copy Approved for Release|201 1/01/2...  \n",
              "8    \\n\\nDeclassified in Part - Sanitized Eopy App...  \n",
              "9   Approved For Release 2009/10/06 : CIA-RDP86B00...  \n",
              "10   \\n\\n25X1\\nApproved For Release 2008/10/16 : C...  \n",
              "11   \\n\\n25X1\\nSanitized Copy Approved for Release...  "
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NkHsSKkmFEz5"
      },
      "source": [
        "Try other calls and process the dataframe as you see fit"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyMrLjKKv2OheIWijMQomGKM",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}