func_nb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_date_from_str(str_date):\n",
    "    return datetime.strptime(str_date, \"%Y%m%d\")\n",
    "\n",
    "\n",
    "def get_bus_info_path(date, list_info_files):\n",
    "    \n",
    "    def date_to_path(date):\n",
    "        return date.strftime(\"/%Y/%m/%d\")\n",
    "    \n",
    "    path = pathlib.Path(BUSINFO_PATH + date_to_path(date))\n",
    "    \n",
    "    file_path = {\n",
    "        p.name: str(p)\n",
    "        for p \n",
    "        in path.iterdir()\n",
    "        if p.name in list_info_files\n",
    "    }\n",
    "    \n",
    "    for l in list_info_files:\n",
    "        if not file_path[l]:\n",
    "            file_path[l] = None\n",
    "    \n",
    "    return file_path\n",
    "\n",
    "\n",
    "def get_prediction_path(date):\n",
    "    def date_to_file_name(date):\n",
    "        return date.strftime(\"/%Y_%m_%d.csv\")\n",
    "    \n",
    "    path = pathlib.Path(PREDICTION_PATH + date_to_file_name(date))\n",
    "    \n",
    "    return str(path) if path.exists() else None\n",
    "\n",
    "\n",
    "def read_dim_data(path, is_list_json=True):\n",
    "    if is_list_json:\n",
    "        vars_data = pd.read_json(path, lines=True).stack().reset_index(drop=True)\n",
    "\n",
    "        vars_df = pd.io.json.json_normalize(vars_data[~vars_data.isna()])\n",
    "    else:\n",
    "        vars_df = pd.read_json(path, lines=True)\n",
    "    \n",
    "    return vars_df\n",
    "\n",
    "\n",
    "def expand_column(stops_df, column):\n",
    "    stops_df = stops_df.reset_index()\n",
    "\n",
    "    stops_stag = pd.concat(\n",
    "        [pd.DataFrame(d)\n",
    "         for d \n",
    "         in stops_df[column]], \n",
    "        keys=stops_df[\"index\"]\n",
    "    ).reset_index(level=1, drop=True)\n",
    "\n",
    "    stops_df = (\n",
    "        stops_df.drop([\"index\"], axis=1)\n",
    "            .join(stops_stag)\n",
    "            .reset_index(drop=True)\n",
    "    )\n",
    "    \n",
    "    return stops_df\n",
    "\n",
    "\n",
    "def read_paths(path):\n",
    "    paths_df = (\n",
    "        read_dim_data(path, False)\n",
    "            .set_index([\"RouteId\", \"RouteVarId\"])\n",
    "            .apply(pd.Series.explode)\n",
    "            .reset_index()\n",
    "    )\n",
    "    \n",
    "    paths_df.loc[:, [\"lat\", \"lng\"]] = paths_df[[\"lat\", \"lng\"]].astype(float)\n",
    "    \n",
    "    return paths_df\n",
    "\n",
    "\n",
    "def read_stops(path):\n",
    "    stops_df = expand_column(\n",
    "        read_dim_data(path, False),\n",
    "        \"Stops\"  # Column contains Stop data\n",
    "    )\n",
    "    \n",
    "    return stops_df\n",
    "\n",
    "\n",
    "def copy_data(data):\n",
    "    \n",
    "    return {key: value.copy() for key, value in data.items()}\n",
    "\n",
    "\n",
    "def upper_columns_names(data):\n",
    "    for k, v in data.items():\n",
    "        v.columns = v.columns.str.upper()\n",
    "\n",
    "\n",
    "def load_predictions(path, column_headers):\n",
    "    print(path)\n",
    "    return pd.read_csv(\n",
    "        path,\n",
    "        dtype=\"str\",\n",
    "        names=column_headers\n",
    "    ).drop(\"UNAMED\", axis=1)\n",
    "\n",
    "\n",
    "def preprocessing_prediction(predic):\n",
    "    predic_df = predic.copy()\n",
    "    \n",
    "    int_columns = [\"STOPID\", \"ROUTEVARID\", \"ROUTEID\"]\n",
    "    float_columns = [\"DISTANCE\", \"SPEED\", \"TIMETOSTOP\"]\n",
    "    \n",
    "    predic_df[int_columns] = predic_df.loc[:, int_columns].astype(\"float\").astype(\"int\")\n",
    "    predic_df[float_columns] = predic_df.loc[:, float_columns].astype(\"float\")\n",
    "    \n",
    "    return predic_df\n",
    "\n",
    "\n",
    "def get_stops_full_data(all_data):\n",
    "    vars_pre = all_data[\"vars.json\"].merge(\n",
    "        all_data[\"routes.json\"][[\"ROUTEID\", \"ROUTENAME\"]],\n",
    "        on=\"ROUTEID\",\n",
    "        how=\"left\",\n",
    "        suffixes=(\"\", \"_y\")\n",
    "    )\n",
    "    \n",
    "    all_data[\"full_vars\"] = vars_pre\n",
    "    \n",
    "    stop_pre = all_data[\"stops.json\"].merge(\n",
    "        vars_pre,\n",
    "        on=[\"ROUTEID\", \"ROUTEVARID\"],\n",
    "        how=\"left\",\n",
    "    ).drop(\"STOPS\", axis=1).drop_duplicates()\n",
    "    \n",
    "    all_data[\"full_stops\"] = stop_pre\n",
    "    \n",
    "\n",
    "def get_full_prediction_data(predic, stop_data):\n",
    "    example = predic.copy()\n",
    "    example_full = example.merge(\n",
    "        stop_data,\n",
    "        on=[\"STOPID\", \"ROUTEID\", \"ROUTEVARID\"],\n",
    "        how=\"left\",\n",
    "        suffixes=(\"\", \"_DIM\")\n",
    "    )\n",
    "    \n",
    "    return example_full\n",
    "\n",
    "# List of functions to load data\n",
    "BUS_INFO_FUNC = MappingProxyType({\n",
    "    \"timetables.json\": functools.partial(read_dim_data, is_list_json=True),\n",
    "    \"routes.json\": functools.partial(read_dim_data, is_list_json=True),\n",
    "    \"vars.json\": functools.partial(read_dim_data, is_list_json=True),\n",
    "    \"trips.json\": functools.partial(read_dim_data, is_list_json=True),\n",
    "    \"stops.json\": functools.partial(\n",
    "        read_stops\n",
    "    ),\n",
    "    \"paths.json\": functools.partial(\n",
    "        read_paths\n",
    "    ),\n",
    "})\n",
    "\n",
    "\n",
    "def get_all_bus_info(date, list_info_files):\n",
    "    bus_data = {}\n",
    "    \n",
    "    bus_info_path = get_bus_info_path(date, list_info_files)\n",
    "    \n",
    "    for key, value in bus_info_path.items():\n",
    "        print(f\"Get data from {value}\")\n",
    "        bus_data[key] = BUS_INFO_FUNC[key](path=value)\n",
    "        \n",
    "    return bus_data\n",
    "\n",
    "\n",
    "def remove_missing_records_on_column(data, cols):\n",
    "    missing_rec_mask = data[cols].isna().any(axis=1)\n",
    "    \n",
    "    return data[~missing_rec_mask].copy()\n",
    "\n",
    "\n",
    "def get_distance_ratio_each_route(stops_ddf):\n",
    "    stops_distance = stops_ddf.copy()\n",
    "\n",
    "    stops_distance = stops_distance.merge(\n",
    "        stops_distance.groupby([\"ROUTEID\", \"ROUTEVARID\"]).RANK.max().to_frame(\"MAXRANK\"),\n",
    "        on=[\"ROUTEID\", \"ROUTEVARID\"],\n",
    "        how=\"left\",\n",
    "    )\n",
    "\n",
    "    # For First Point\n",
    "    pri_keys = [\"ROUTEID\", \"ROUTEVARID\"]\n",
    "    value_cols = [\"SPOINT\"]\n",
    "    first_stops = stops_distance.loc[stops_distance.RANK==1, pri_keys + value_cols].copy()\n",
    "    stops_distance = stops_distance.merge(\n",
    "        first_stops,\n",
    "        on=pri_keys,\n",
    "        how=\"left\",\n",
    "        suffixes=(\"\", \"FSPOINT\")\n",
    "    )\n",
    "\n",
    "    # For Last Point\n",
    "    pri_keys = [\"ROUTEID\", \"ROUTEVARID\"]\n",
    "    value_cols = [\"SPOINT\"]\n",
    "    first_stops = stops_distance.loc[stops_distance.RANK==stops_distance.MAXRANK, pri_keys + value_cols].copy()\n",
    "    stops_distance = stops_distance.merge(\n",
    "        first_stops,\n",
    "        on=pri_keys,\n",
    "        how=\"left\",\n",
    "        suffixes=(\"\", \"LSPOINT\")\n",
    "    )\n",
    "\n",
    "    def _change_slice(rec):\n",
    "        new_linestr = list(rec[\"LINESTRING\"].coords)\n",
    "        new_linestr[0] = rec[\"SPOINTFSPOINT\"].coords[0]\n",
    "        new_linestr[-1] = rec[\"SPOINTLSPOINT\"].coords[0]\n",
    "\n",
    "        return LineString(new_linestr)\n",
    "\n",
    "    _ = stops_distance.apply(lambda x: _change_slice(x), axis=1)\n",
    "    stops_distance[\"STOPS_LINESTRING\"] = _\n",
    "    stops_distance[\"STOPSDISTANCE\"] = stops_distance.apply(lambda x: x[\"STOPS_LINESTRING\"].project(x[\"SPOINT\"]), axis=1)\n",
    "    stops_distance[\"ALLSTOPSDISTANCE\"] = stops_distance.apply(lambda x: x[\"STOPS_LINESTRING\"].project(x[\"SPOINTLSPOINT\"]), axis=1)\n",
    "    stops_distance[\"RATIOSTOPSDISTANCE\"] = stops_distance.eval(\"STOPSDISTANCE / ALLSTOPSDISTANCE\")\n",
    "    \n",
    "    return stops_distance\n",
    "\n",
    "\n",
    "def get_log_distance_ratio_predict(predic_df):\n",
    "    # Convert to integer for easier to predict\n",
    "    predic_df[\"TIMESTAMP\"] = pd.to_datetime(predic_df[\"LOCTIMESTAMP\"]).astype(\"int64\") // 10**9\n",
    "\n",
    "    # preprocessing some columns in predic data\n",
    "    predic_df[\"BUSIDSTR\"] = predic_df[\"BUSID\"].str.strip()  #remove space\n",
    "    predic_df[\"LOCTIMESTAMP\"] = pd.to_datetime(predic_df[\"LOCTIMESTAMP\"])  #convert to datetime\n",
    "    predic_df[\"STOPIDSTR\"] = predic_df[\"STOPID\"].astype(\"str\")  #convert to string\n",
    "    predic_df[\"DISTANCERATIO\"] = predic_df.eval(\"DISTANCE / DISTANCE_DIM\")\n",
    "\n",
    "    return predic_df\n",
    "\n",
    "\n",
    "def remove_outlier_whole_trips(df, plot=False):\n",
    "    center_data = df[df.FIRSTHALF==99].copy()\n",
    "    \n",
    "    data_half = {\n",
    "        \"first\": df[df.FIRSTHALF==1].copy(),\n",
    "        \"center\": center_data,\n",
    "        \"second\": df[df.FIRSTHALF==2].copy(),\n",
    "    }\n",
    "    \n",
    "    def _mark_outlier(data, is_reverse):\n",
    "        outlier = mark_all_velocity_outlier(data[\"DISTANCE\"], data[\"TIMESTAMP\"], is_reverse=is_reverse, threshold=1)\n",
    "\n",
    "        # Show detected outliers\n",
    "        data[\"VREMOVE\"] = outlier\n",
    "\n",
    "        if plot:\n",
    "            fig, ax = plt.subplots(figsize=(10, 8))\n",
    "            _draw_time_distance(data, hue=\"VREMOVE\", alpha=0.8)\n",
    "            plt.show()\n",
    "\n",
    "            # data after remove outliers\n",
    "            fig, ax = plt.subplots(figsize=(10, 8))\n",
    "            _draw_time_distance(data[data.VREMOVE==0], alpha=0.8)\n",
    "            plt.show()\n",
    "    \n",
    "        return data[data.VREMOVE==0].copy()\n",
    "    \n",
    "    dt = []\n",
    "    for i, v in data_half.items():\n",
    "        if i == \"center\":\n",
    "            dt.append(v)\n",
    "        else:\n",
    "            is_rv = True if i == \"first\" else False\n",
    "            dt.append(_mark_outlier(v, is_rv))\n",
    "            \n",
    "    df = pd.concat(dt, axis=0, sort=False)\n",
    "\n",
    "    return df\n",
    "\n",
    "def remove_outliers_whole_route(df, plot=False):\n",
    "    trips = df.TRIPS.unique()\n",
    "    print(\"There are {} trips: {}\".format(trips.shape[0], \", \".join(trips)))\n",
    "    \n",
    "    rs = []\n",
    "    for t in trips:\n",
    "        print(f\"Remove outliers for trips: {t}\")\n",
    "        dt = df[df.TRIPS==t].copy()\n",
    "        rs.append(remove_outlier_whole_trips(dt, plot))\n",
    "        \n",
    "    return pd.concat(rs, sort=False)\n",
    "\n",
    "def remove_distance_not_increase(ds, n_iters=500):\n",
    "    ds = ds.copy()\n",
    "    msk = pd.Series(False, index=ds.index)\n",
    "    \n",
    "    for i in range(n_iters):\n",
    "        delta_d = ds.shift() - ds\n",
    "        invalid = delta_d <= 0\n",
    "        if invalid.sum() <= 0:\n",
    "            print(\"There is no decrease distance\")\n",
    "            break\n",
    "        msk |= invalid\n",
    "        ds = ds[~msk].copy()\n",
    "    \n",
    "    return msk.copy()\n",
    "\n",
    "def interpolate_time(predic_df, stops_df, plot=False):\n",
    "    msk = remove_distance_not_increase(predic_df.DISTANCE.copy())\n",
    "    \n",
    "    predic_df = predic_df[~msk].copy()\n",
    "    x = 1 - predic_df[\"DISTANCERATIO\"]\n",
    "    y = predic_df[\"TIMESTAMP\"].astype(\"int\")\n",
    "    y = y[y.index.isin(x.index)].copy()\n",
    "    spl = InterpolatedUnivariateSpline(x, y, k=1)\n",
    "    xs = stops_df[\"RATIOSTOPSDISTANCE\"]\n",
    "    if plot:\n",
    "        fig, ax = plt.subplots(figsize=(10, 8))\n",
    "        plt.plot(xs, spl(xs), 'gv')\n",
    "        plt.plot(x, y, 'ro', ms=5, alpha=0.2, linewidth=6)\n",
    "        plt.show()\n",
    "\n",
    "    return spl(xs)\n",
    "\n",
    "def predic_whole_route(route_data, stops, plot, cols=None, pref=\"ARRIVETIMETRIPS\"):\n",
    "    trips = route_data.TRIPS.unique()\n",
    "    stops = stops.copy()\n",
    "\n",
    "    ipl = []\n",
    "    for t in trips:\n",
    "        print(f\"Predict for trips {t}\")\n",
    "        dta = route_data[route_data.TRIPS==t].copy()\n",
    "        ipl.append(interpolate_time(dta, stops, plot))\n",
    "        col_names = []\n",
    "        \n",
    "    for i, v in enumerate(ipl):\n",
    "        cn = pref + \"{:02d}\".format(int(i))\n",
    "        col_names.append(cn)\n",
    "        stops[cn] = v\n",
    "    \n",
    "    col_pre = stops.columns[stops.columns.str.contains(pref)].tolist()\n",
    "    for c in col_pre:\n",
    "        print(f\"Convert column name {c}\")\n",
    "        stops[c] = convert_timezone(stops[c])\n",
    "     \n",
    "    if cols:\n",
    "        dt_rt = stops[cols + col_pre].copy()\n",
    "    else:\n",
    "        dt_rt = stops.copy()\n",
    "    \n",
    "    return dt_rt\n",
    "\n",
    "\n",
    "def predict_whole_route_all_bus(predic, \n",
    "                                stops, \n",
    "                                route_id, \n",
    "                                routevar_id, \n",
    "                                cols,\n",
    "                                plot=(False, False),\n",
    "                                ratio_distance=0.8):\n",
    "    \n",
    "    stops_df = stops[(stops.ROUTEID==route_id) & (stops.ROUTEVARID==routevar_id)].copy()\n",
    "    predic_df = predic[(predic.ROUTEID==route_id) & (predic.ROUTEVARID==routevar_id)].copy()\n",
    "    \n",
    "    bus = predic_df.BUSID.unique()\n",
    "    \n",
    "    time_tables = []\n",
    "    for b in bus:\n",
    "        try:\n",
    "            bus_data = predic_df[predic_df.BUSID==b].copy()\n",
    "            print(f\"Predic for bus {b} with shape {bus_data.shape}\")\n",
    "            dt_removed = preprocessing_route_bus(bus_data, ratio_distance)\n",
    "            dt_removed = remove_outliers_whole_route(dt_removed, plot[0])\n",
    "            print(f\"Shape after preprocessing {dt_removed.shape}\")\n",
    "\n",
    "            predicted = predic_whole_route(dt_removed, stops_df, plot[1], cols=cols)\n",
    "            predicted[\"BUSPREDIC\"] = b\n",
    "        \n",
    "            time_tables.append(predicted.copy())\n",
    "        except Exception as e:\n",
    "            print(str(e))\n",
    "            pass\n",
    "\n",
    "    return pd.concat(time_tables, axis=0, sort=True, ignore_index=True)\n",
    "\n",
    "\n",
    "def mark_first_half(df, rate=(0.1, 0.9)):\n",
    "    df[\"FIRSTHALF\"] = np.nan\n",
    "    df.loc[df.DISTANCE < (df.DISTANCE_DIM.max() * rate[0]), \"FIRSTHALF\"] = 2\n",
    "    df.loc[df.DISTANCE > (df.DISTANCE_DIM.max() * rate[1]), \"FIRSTHALF\"] = 1\n",
    "    df.loc[df.FIRSTHALF.isna(), \"FIRSTHALF\"] = 99\n",
    "\n",
    "\n",
    "def preprocessing_route_bus(df, ratio_distance_peak=0.8):\n",
    "    df = df.drop_duplicates([\"TIMESTAMP\"], keep=\"first\").copy()\n",
    "    df = df.sort_values([\"TIMESTAMP\"]).copy()\n",
    "    df[\"IS_PEAK\"] = mark_peak_point(df, df.DISTANCE.max(), ratio_distance_peak)\n",
    "    df[\"TRIPS\"] = split_trips(df, xffix=(\"TRIPS\", \"\"))\n",
    "    df = df.dropna(subset=[\"TRIPS\"]).copy()\n",
    "    mark_first_half(df)\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "def mark_peak_point(df, distance, ratio=1):\n",
    "    dis = df[\"DISTANCE\"].copy()\n",
    "    \n",
    "    shifted_dis = dis - dis.shift()\n",
    "    msk_peak = shifted_dis > (distance * ratio)\n",
    "    \n",
    "    return msk_peak\n",
    "\n",
    "\n",
    "# Tách các trips\n",
    "def split_trips(df, cols=\"IS_PEAK\", xffix=(\"\", \"\")):\n",
    "    trips = df[cols].cumsum().copy().astype(\"int\").to_frame(\"PEAK\")\n",
    "    \n",
    "    mean_log = trips.groupby(\"PEAK\").size().mean() * 0.5\n",
    "    drop_peak = trips.groupby(\"PEAK\").size()\n",
    "    drop_peak = drop_peak[drop_peak < mean_log]\n",
    "    trips = trips[~trips.PEAK.isin(drop_peak.index.tolist())].copy()\n",
    "    trips.PEAK = (trips.PEAK.rank(method=\"dense\") - 1).astype(\"int\").astype(\"str\")\n",
    "    trips = trips.PEAK\n",
    "    \n",
    "    trips = xffix[0] + trips.str.zfill(2) + xffix[1]\n",
    "    \n",
    "    return trips\n",
    "\n",
    "\n",
    "def get_velocity(dfd, dft, is_reverse=False):\n",
    "    if is_reverse:\n",
    "        d = dfd[::-1].copy()\n",
    "        t = dft[::-1].copy()\n",
    "        dd = d - d.shift()\n",
    "    else:\n",
    "        d = dfd.copy()\n",
    "        t = dft.copy()\n",
    "        dd = d.shift() - d\n",
    "    \n",
    "\n",
    "    tt = (t - t.shift()).abs()\n",
    "    \n",
    "    return (dd / tt)[::-1] if is_reverse else (dd / tt)\n",
    "\n",
    "\n",
    "def mark_velocity_outlier(v_col, threshold=1.5):\n",
    "    msk_vel = v_col < threshold\n",
    "    msk_rm = pd.Series(False, index=v_col.index)\n",
    "    msk_rm[msk_vel] = True\n",
    "    \n",
    "    return msk_rm.copy()\n",
    "\n",
    "\n",
    "def mark_all_velocity_outlier(dfd, dft, threshold=1, is_reverse=False, n_iters=5):\n",
    "    index = dfd.index.copy()\n",
    "    msk_vel = pd.Series(False, index=index)\n",
    "    \n",
    "    print(f\"Starting to remove outlier with velocity threshold {threshold} \"\n",
    "          f\"and max number of iterates is {n_iters} times\")\n",
    "    for i in range(n_iters):\n",
    "        velocity = get_velocity(dfd[~msk_vel], dft[~msk_vel], is_reverse)\n",
    "        start_msk_vel = mark_velocity_outlier(velocity, threshold)\n",
    "        num_outlier = start_msk_vel.sum()\n",
    "        print(f\"Detected {num_outlier} outlier.\")\n",
    "        \n",
    "        if num_outlier <= 0:  # exit loop if there is no outlier\n",
    "            print(\"There is no outliers remain!\")\n",
    "            break\n",
    "            \n",
    "        dfd = dfd[~start_msk_vel].copy()\n",
    "        dft = dft[~start_msk_vel].copy()\n",
    "        \n",
    "        msk_vel |= start_msk_vel\n",
    "\n",
    "    return msk_vel\n",
    "\n",
    "\n",
    "def _draw_time_distance(df, cols=(\"TIMESTAMP\", \"DISTANCE\"), hue=None, alpha=0.2, size=None):\n",
    "    # Plot example\n",
    "    sns.scatterplot(data=df, x=cols[0], y=cols[1], hue=hue, alpha=alpha)\n",
    "    \n",
    "\n",
    "def convert_timezone(sr):\n",
    "    return pd.to_datetime(\n",
    "            sr, \n",
    "            unit=\"s\"\n",
    "        ).dt.tz_localize(\n",
    "            \"UTC\"\n",
    "        ).dt.tz_convert(\"Asia/Ho_Chi_Minh\").dt.tz_localize(None)\n",
    "\n",
    "\n",
    "def load_data(date):\n",
    "    # Load data của các chuyến xe bus.\n",
    "    print(\"Get bus info.\")\n",
    "    all_data_cp = get_all_bus_info(\n",
    "        date, \n",
    "        list_info_files=list_bus_info_file_names\n",
    "    )\n",
    "    # Upper all column names\n",
    "    upper_columns_names(all_data_cp)\n",
    "    # Lấy thông tin (merge thông tin) từ nhiều file\n",
    "    get_stops_full_data(all_data_cp)\n",
    "    \n",
    "    # read predictions file\n",
    "    print(\"Get log data.\")\n",
    "    prediction_df = load_predictions(\n",
    "        get_prediction_path(date),\n",
    "        PREDICTION_COLUMN_HEADERS\n",
    "    )\n",
    "    # convert some columns from str to int, float or datetime\n",
    "    prediction_df = preprocessing_prediction(prediction_df)\n",
    "    # Lấy thông tin của Stops vào Prediction\n",
    "    full_predic = get_full_prediction_data(prediction_df, all_data_cp[\"full_stops\"])\n",
    "    full_predic = remove_missing_records_on_column(full_predic, [\"CODE\"])\n",
    "    full_predic = get_log_distance_ratio_predict(full_predic)\n",
    "    \n",
    "    return all_data_cp, full_predic\n",
    "\n",
    "\n",
    "def get_linestring_all_path(path):\n",
    "    paths_df = path.dropna().copy()\n",
    "\n",
    "    paths_df[\"POINT\"] = paths_df.apply(lambda x: Point(x[\"LNG\"], x[\"LAT\"]), axis=1)\n",
    "\n",
    "    lines_df = (\n",
    "        paths_df.groupby([\"ROUTEID\", \"ROUTEVARID\"])\n",
    "            .POINT.apply(list)\n",
    "            .apply(LineString)\n",
    "            .to_frame(\"LINESTRING\")\n",
    "            .reset_index()\n",
    "    )\n",
    "    \n",
    "    return lines_df\n",
    "\n",
    "\n",
    "def get_data_processed(date):    \n",
    "    all_data_cp, full_predic = load_data(date)\n",
    "    \n",
    "    # Rank data stops for each route\n",
    "    print(\"Ranking stops sequences.\")\n",
    "    line_string = get_linestring_all_path(all_data_cp[\"paths.json\"])\n",
    "    stops_ddf = all_data_cp[\"stops.json\"].copy()\n",
    "    stops_ddf = stops_ddf.dropna(subset=[\"LNG\", \"LAT\"]).copy()\n",
    "    stops_ddf = stops_ddf.merge(\n",
    "        line_string,\n",
    "        on=[\"ROUTEID\", \"ROUTEVARID\"],\n",
    "        how=\"inner\",\n",
    "        suffixes=(\"\", \"LINEINFO\")\n",
    "    )\n",
    "\n",
    "    stops_ddf[\"SPOINT\"] = stops_ddf.apply(\n",
    "        lambda x: Point(x[\"LNG\"], x[\"LAT\"]), axis=1\n",
    "    )\n",
    "    stops_ddf[\"DISTANCE\"] = stops_ddf.apply(\n",
    "        lambda x: x[\"LINESTRING\"].project(x[\"SPOINT\"]), \n",
    "        axis=1\n",
    "    )\n",
    "    stops_ddf[\"NEARESTPOINT\"] = stops_ddf.apply(\n",
    "        lambda x: nearest_points(x[\"LINESTRING\"], x[\"SPOINT\"])[0],\n",
    "        axis=1,\n",
    "    )\n",
    "    stops_ddf[\"RANK\"] = stops_ddf.groupby([\"ROUTEID\", \"ROUTEVARID\"]).DISTANCE.rank(method=\"first\")\n",
    "\n",
    "    # final data with ranking of stops id and distance ratio\n",
    "    stops_ddf = stops_ddf.sort_values([\"ROUTEID\", \"ROUTEVARID\", \"RANK\"])\n",
    "    print(\"Get Ratio distance for each stops.\")\n",
    "    stops_ddf = get_distance_ratio_each_route(stops_ddf).drop(\"STOPS\", axis=1)\n",
    "    \n",
    "    return stops_ddf, full_predic\n",
    "\n",
    "\n",
    "def generate_date_from_range(from_date, to_date):\n",
    "    curr_date = from_date\n",
    "    while curr_date <= to_date:\n",
    "        yield curr_date\n",
    "        curr_date = curr_date + timedelta(days=1)\n",
    "\n",
    "        \n",
    "def run_predict_all_route(pre, stop, list_route, plot=(False, False)):\n",
    "    \n",
    "    result = []\n",
    "    for lr in list_route:\n",
    "        print(\"Running for ROUTEID {} and ROUTEVARID {}\".format(\n",
    "            lr[0],\n",
    "            lr[1],\n",
    "        ))\n",
    "        \n",
    "        time_tables = predict_whole_route_all_bus(\n",
    "            pre, \n",
    "            stop, \n",
    "            route_id=lr[0], \n",
    "            routevar_id=lr[1], \n",
    "            plot=plot,\n",
    "            cols=cols_to_get, \n",
    "            ratio_distance=0.8\n",
    "        )\n",
    "\n",
    "        result.append(time_tables)\n",
    "    \n",
    "    return pd.concat(result, axis=0, sort=True, ignore_index=True)\n",
    "         \n",
    "\n",
    "def run_pipeline_multi_date(date_list, route_id=None, route_var_id=None, plot=(False, False)):\n",
    "    \n",
    "    result = {}\n",
    "    for d in date_list:\n",
    "        date_str = d.strftime(\"%Y-%m-%d\")\n",
    "        stop, pre = get_data_processed(d)\n",
    "        \n",
    "        if route_id and route_var_id:\n",
    "            list_route = [(route_id, route_var_id)]\n",
    "        else:\n",
    "            list_route = (\n",
    "                pre[[\"ROUTEID\", \"ROUTEVARID\"]]\n",
    "                    .drop_duplicates()\n",
    "                    .values.tolist()\n",
    "            )\n",
    "        \n",
    "        cols_to_get = [\"ROUTEID\", \"ROUTEVARID\", \n",
    "                       \"STOPID\", \"CODE\", \"NAME\", \"STOPTYPE\", \n",
    "                       \"ZONE\", \"WARD\", \"ADDRESSNO\", \"STREET\", \"SUPPORTDISABILITY\", \n",
    "                       \"STATUS\", \"LNG\", \"LAT\", \n",
    "                       \"SEARCH\", \"ROUTES\", \"RANK\"]\n",
    "        print(\"Predict for date {}\".format(date_str))\n",
    "        \n",
    "        time_tables = run_predict_all_route(\n",
    "            pre,\n",
    "            stop,\n",
    "            list_route,\n",
    "            plot,\n",
    "        )\n",
    "\n",
    "        result[date_str] = time_tables\n",
    "        \n",
    "    return result\n",
    "\n",
    "\n",
    "def plot_route_stops(df, figsize=(20, 20), show_path=(True, True), show_real_stops=True, show_nearest_point=True,\n",
    "                     annotation=False):\n",
    "    geo_df = gpd.GeoDataFrame(df, geometry=\"SPOINT\")\n",
    "    # Example plot result\n",
    "    fig, ax = plt.subplots(figsize=figsize)\n",
    "    ax.grid(False)\n",
    "    \n",
    "    if any(show_path):\n",
    "        if show_path[0]:\n",
    "            gpd.GeoSeries(df.LINESTRING.values[0]).plot(ax=ax, figure=fig)\n",
    "        if show_path[1]:\n",
    "            gpd.GeoSeries([Point(i) for i in plot_example.LINESTRING.values[0].coords]).plot(\n",
    "                ax=ax,\n",
    "                figure=fig,\n",
    "                color=\"b\"\n",
    "            )\n",
    "            \n",
    "    # stop_gpd.plot(ax=ax, figure=fig, color=\"r\", marker=\"v\")\n",
    "    if annotation:\n",
    "        for i, x in geo_df.iterrows():\n",
    "            plt.annotate(s=x[\"RANK\"], xy=(x[\"LNG\"], x[\"LAT\"]), horizontalalignment=\"right\")\n",
    "    \n",
    "    if show_real_stops:\n",
    "        df.set_geometry(\"SPOINT\").copy().plot(ax=ax, figure=fig, color=\"r\", alpha=0.8)\n",
    "    \n",
    "    if show_nearest_point:\n",
    "        df.set_geometry(\"NEARESTPOINT\").copy().plot(ax=ax, figure=fig, color=\"g\", alpha=1, marker=\"v\")\n",
    "    \n",
    "    list_route = df[[\"ROUTEID\", \"ROUTEVARID\"]].drop_duplicates().values.tolist()\n",
    "    \n",
    "    \n",
    "def _draw(route_id, route_var_id, date):\n",
    "    str_date = date.strftime(\"%Y%m%d\")\n",
    "    plt.title(\"ROUTEID={}, ROUTEVARID={}, DATE={}\".format(\n",
    "        route_id,\n",
    "        route_var_id,\n",
    "        str_date,\n",
    "    ))\n",
    "    plt.show()\n",
    "\n",
    "    \n",
    "def plot_kde_stops(example_stop_list, stops_df, save_folder, plot=False):\n",
    "    def _plot_wrapper(pl):\n",
    "        if pl:\n",
    "            plt.show()\n",
    "        else:\n",
    "            pass\n",
    "    \n",
    "    for s in example_stop_list:\n",
    "        print(f\"Save fig/Show for stop: {s}\")\n",
    "        example_stop_plot = stops_df[stops_df.STOPID == s].copy()\n",
    "        stop_id = \"ST_\" + str(s)\n",
    "        example_stop_plot[\"STOPCODE\"] = \"ST: \" + example_stop_plot.STOPID.astype(\"str\")\n",
    "        example_stop_plot = example_stop_plot.query(\"DELTADIS > 0\").copy()\n",
    "\n",
    "        # Multi KDE\n",
    "        grid = sns.FacetGrid(example_stop_plot, hue=\"STOPCODE\", height=10)\n",
    "        grid.map(sns.kdeplot, \"DELTADIS\")\n",
    "        grid.add_legend()\n",
    "        plt.savefig(fname=f\"{save_folder}/{stop_id}_KDE.svg\", format=\"svg\", dpi=1200)\n",
    "        _plot_wrapper(plot)\n",
    "\n",
    "        # Plot kde of DELTADIS\n",
    "        fig, ax = plt.subplots(figsize=(16, 8))\n",
    "        sns.violinplot(y=\"DELTADIS\", x=\"STOPCODE\", ax=ax, data=example_stop_plot)\n",
    "        plt.savefig(fname=f\"{save_folder}/{stop_id}_VIOLIN.svg\", format=\"svg\", dpi=1200)\n",
    "        _plot_wrapper(plot)\n",
    "\n",
    "        # Plot kde of DELTADIS\n",
    "        fig, ax = plt.subplots(figsize=(16, 8))\n",
    "        sns.boxplot(y=\"DELTADIS\", x=\"STOPCODE\", ax=ax, data=example_stop_plot)\n",
    "        plt.savefig(fname=f\"{save_folder}/{stop_id}_BOXPLOT.svg\", format=\"svg\", dpi=1200)\n",
    "        _plot_wrapper(plot)\n",
    "        \n",
    "        plt.close(\"all\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 (ltpl)",
   "language": "python",
   "name": "base"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}