diff --git a/evaluate/evaluate_data_set_coding.ipynb b/evaluate/evaluate_data_set_coding.ipynb index 2690efc..321ba0b 100644 --- a/evaluate/evaluate_data_set_coding.ipynb +++ b/evaluate/evaluate_data_set_coding.ipynb @@ -7,7 +7,7 @@ }, "source": [ "

Table of Contents

\n", - "
" + "
" ] }, { @@ -903,7 +903,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Process JSON\n", + "# Evaluate all publications\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Process JSON\n", "\n", "- Back to [Table of Contents](#Table-of-Contents)" ] @@ -925,7 +934,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Filter publications?\n", + "### Filter publications?\n", "\n", "- Back to [Table of Contents](#Table-of-Contents)\n", "\n", @@ -936,7 +945,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### django init\n", + "#### django init\n", "\n", "- Back to [Table of Contents](#Table-of-Contents)" ] @@ -967,7 +976,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### get IDs of included publications\n", + "#### get IDs of included publications\n", "\n", "- Back to [Table of Contents](#Table-of-Contents)" ] @@ -1079,7 +1088,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## process JSON\n", + "### process JSON files\n", "\n", "- Back to [Table of Contents](#Table-of-Contents)" ] @@ -1136,15 +1145,6 @@ "#-- END if output... --#\n" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Evaluate\n", - "\n", - "- Back to [Table of Contents](#Table-of-Contents)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1641,6 +1641,9 @@ }, "outputs": [], "source": [ + "# set precision_recall_graph_path\n", + "#precision_recall_graph_path = \"{}/{}precision_recall_graph{}.pdf\".format( output_folder_path, file_name_prefix, file_name_suffix )\n", + "\n", "# declare variables\n", "plot_details = None\n", "\n", @@ -1685,6 +1688,911 @@ }, "outputs": [], "source": [ + "# set results file path:\n", + "#results_file_path = \"{}/{}evaluation_results{}.txt\".format( output_folder_path, file_name_prefix, file_name_suffix )\n", + "\n", + "# declare variables\n", + "results_file = None\n", + "line_list_string = None\n", + "\n", + "# do we output to file?\n", + "if ( output_to_file == True ):\n", + " \n", + " if ( debug_flag == True ):\n", + " print( line_list )\n", + " #-- END check to see if debug --#\n", + " \n", + " # yes. open output file.\n", + " with open( results_file_path, mode = \"w\" ) as results_file:\n", + "\n", + " # join line list with \"/n\", then write.\n", + " line_list_string = \"\\n\".join( line_list )\n", + " results_file.write( line_list_string )\n", + "\n", + " #-- END with...as --#\n", + " \n", + " print( \"results output to {}\".format( results_file_path ) )\n", + " \n", + "#-- END check to see if we output to file --#\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Evaluate only publications with citations\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)\n", + "\n", + "Now, try just doing this on publications that have citations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# reset line_list.\n", + "line_list = []\n", + "\n", + "# if output...\n", + "output_string = \"Using baseline/ground_truth file: {}\".format( baseline_json_path )\n", + "print( output_string )\n", + "\n", + "if ( output_to_file == True ):\n", + " \n", + " # store line for output\n", + " line_list.append( output_string )\n", + "\n", + "#-- END if output to file... --#" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Process JSON\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# init class to handle evaluation\n", + "coding_evaluator = CitationCodingEvaluation()\n", + "coding_evaluator.debug_flag = debug_flag" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Filter publications?\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)\n", + "\n", + "If we want to only run on a subset of publications, need to use some django code." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### django init\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "path_to_django_init = \"/home/context/django_init.py\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "%run $path_to_django_init" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### get IDs of included publications\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "from sourcenet.models import Article" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Working with django-taggit**\n", + "\n", + "- django-taggit documentation: https://github.com/alex/django-taggit\n", + "\n", + "Adding tags to a model:\n", + "\n", + " from django.db import models\n", + " \n", + " from taggit.managers import TaggableManager\n", + " \n", + " class Food(models.Model):\n", + " # ... fields here\n", + " \n", + " tags = TaggableManager()\n", + " \n", + "Interacting with a model that has tags:\n", + "\n", + " >>> apple = Food.objects.create(name=\"apple\")\n", + " >>> apple.tags.add(\"red\", \"green\", \"delicious\")\n", + " >>> apple.tags.all()\n", + " [, , ]\n", + " >>> apple.tags.remove(\"green\")\n", + " >>> apple.tags.all()\n", + " [, ]\n", + " >>> Food.objects.filter(tags__name__in=[\"red\"])\n", + " [, ]\n", + " \n", + " # include only those with certain tags.\n", + " #tags_in_list = [ \"prelim_unit_test_001\", \"prelim_unit_test_002\", \"prelim_unit_test_003\", \"prelim_unit_test_004\", \"prelim_unit_test_005\", \"prelim_unit_test_006\", \"prelim_unit_test_007\" ]\n", + " tags_in_list = [ \"grp_month\", ]\n", + " if ( len( tags_in_list ) > 0 ):\n", + " \n", + " # filter\n", + " print( \"filtering to just articles with tags: \" + str( tags_in_list ) )\n", + " grp_article_qs = grp_article_qs.filter( tags__name__in = tags_in_list )\n", + " \n", + " #-- END check to see if we have a specific list of tags we want to include --#" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# filter to just those in holdout that have one or more related citations.\n", + "\n", + "# declare variables\n", + "article_tag_include_list = None\n", + "article_tag_exclude_list = None\n", + "article_qs = None\n", + "\n", + "# include...\n", + "article_tag_include_list = []\n", + "article_tag_include_list.append( \"holdout\" )\n", + "\n", + "# exclude...\n", + "article_tag_exclude_list = []\n", + "article_tag_exclude_list.append( \"wc_holdout\" )\n", + "\n", + "# get all articles\n", + "article_qs = Article.objects.all()\n", + "\n", + "# include tags?\n", + "if ( ( article_tag_include_list is not None ) and ( len( article_tag_include_list ) > 0 ) ):\n", + " \n", + " # yes.\n", + " article_qs = article_qs.filter( tags__name__in = article_tag_include_list )\n", + " \n", + "#-- END check to see if include tags? --#\n", + "\n", + "# exclude tags?\n", + "if ( ( article_tag_exclude_list is not None ) and ( len( article_tag_exclude_list ) > 0 ) ):\n", + " \n", + " # yes.\n", + " article_qs = article_qs.exclude( tags__name__in = article_tag_exclude_list )\n", + " \n", + "#-- END check to see if include tags? --#\n", + "\n", + "# filter down to just those where related data set citation id is > 0.\n", + "\n", + "print( \"Filtered to {} Article instances.\".format( article_qs.count() ) )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# declare variables\n", + "article_instance = None\n", + "citation_count = None\n", + "article_with_citations_id_list = None\n", + "\n", + "# now, loop over the publications\n", + "article_with_citations_id_list = []\n", + "for article_instance in article_qs:\n", + " \n", + " # see how many citations.\n", + " citation_count = 0\n", + " citation_count = article_instance.datasetcitation_set.all().count()\n", + " \n", + " if ( citation_count > 0 ):\n", + " \n", + " # add to id list.\n", + " article_with_citations_id_list.append( article_instance.id )\n", + " \n", + " #-- END check to see if has citations --#\n", + " \n", + "#-- END loop over articles. --#\n", + "\n", + "# re-do article_qs, limiting to IDs in our list.\n", + "article_qs = Article.objects.filter( id__in = article_with_citations_id_list )\n", + "\n", + "print( \"Filtered to {} Article instances.\".format( article_qs.count() ) )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# declare variables\n", + "article_id_include_list = None\n", + "article_instance = None\n", + "\n", + "# make list of IDs\n", + "article_id_include_list = []\n", + "for article_instance in article_qs:\n", + " \n", + " # get ID\n", + " article_id = article_instance.id\n", + " \n", + " # add to list\n", + " article_id_include_list.append( article_id )\n", + " \n", + "#-- END loop over matching articles. --#\n", + "\n", + "# store details in coding_evaluator\n", + "coding_evaluator.set_excluded_article_tag_list( article_tag_exclude_list )\n", + "coding_evaluator.set_included_article_tag_list( article_tag_include_list )\n", + "coding_evaluator.set_included_article_id_list( article_id_include_list )\n", + "\n", + "print( \"Including {} publications (include tags: {}; exclude tags {}).\".format( len( article_id_include_list ), article_tag_include_list, article_tag_exclude_list ) )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### process JSON files\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# process baseline JSON\n", + "result_type = CitationCodingEvaluation.RESULT_TYPE_BASELINE\n", + "citation_json = baseline_json\n", + "status = coding_evaluator.process_citation_json( citation_json, result_type )\n", + "\n", + "# output\n", + "output_string = \"Processing status for {} (None = Success!): \\\"{}\\\"\".format( result_type, status )\n", + "print( output_string )\n", + "\n", + "# if output...\n", + "if ( output_to_file == True ):\n", + " \n", + " # store line for output\n", + " line_list.append( output_string )\n", + " \n", + "#-- END if output... --#\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# process derived JSON\n", + "result_type = CitationCodingEvaluation.RESULT_TYPE_DERIVED\n", + "citation_json = derived_json\n", + "status = coding_evaluator.process_citation_json( citation_json, result_type )\n", + "\n", + "# output\n", + "output_string = \"Processing status for {} (None = Success!): \\\"{}\\\"\".format( result_type, status )\n", + "print( output_string )\n", + "\n", + "# if output...\n", + "if ( output_to_file == True ):\n", + " \n", + " # store line for output\n", + " line_list.append( output_string )\n", + " \n", + "#-- END if output... --#\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# create lists\n", + "coding_evaluator.debug_flag = False\n", + "details = coding_evaluator.create_evaluation_lists()\n", + "status = details\n", + "baseline_list = coding_evaluator.get_baseline_list()\n", + "derived_raw_list = coding_evaluator.get_derived_raw_list()\n", + "derived_binary_list = coding_evaluator.get_derived_binary_list()\n", + "publication_id_per_citation_list = coding_evaluator.get_publication_id_list()\n", + "data_set_id_per_citation_list = coding_evaluator.get_data_set_id_list()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## precision, recall, and accuracy\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# calculation methods to include\n", + "calculation_methods = []\n", + "calculation_methods.append( CALCULATION_METHOD_DEFAULT )\n", + "calculation_methods.append( CALCULATION_METHOD_BINARY )\n", + "#calculation_methods.append( CACLULATION_METHOD_MACRO )\n", + "#calculation_methods.append( CALCULATION_METHOD_MICRO )\n", + "#calculation_methods.append( CALCULATION_METHOD_WEIGHTED )\n", + "\n", + "# call function to do work.\n", + "output_dictionary = precision_recall_f1( baseline_list, derived_binary_list, calculation_methods )\n", + "\n", + "# add lines from output to line_list\n", + "line_list = line_list + output_dictionary.get( \"line_list\", [] )\n", + "line_list.append( \"\\n\" )\n", + "\n", + "print( \"----> output dictionary: {}\".format( output_dictionary ) )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# check for excluded articles\n", + "excluded_article_count = details.get( CitationCodingEvaluation.DETAILS_EXCLUDED_ARTICLE_COUNT, None )\n", + "excluded_article_id_list = details.get( CitationCodingEvaluation.DETAILS_EXCLUDED_ARTICLE_ID_LIST, None )\n", + "if ( ( excluded_article_count is not None ) and ( excluded_article_count > 0 ) ):\n", + " \n", + " # add excluded articles details:\n", + " line_list.append( \"\\n\" )\n", + " line_string = \"{} excluded publications: {} \".format( excluded_article_count, excluded_article_id_list )\n", + " line_list.append( line_string )\n", + " print( line_string )\n", + " \n", + "#-- END check for excluded publications. --#" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## precision, recall, and accuracy per publication\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# declare variables\n", + "pub_debug_flag = False\n", + "publication_to_lists_map = coding_evaluator.get_lists_by_publication()\n", + "pub_publication_id_list = []\n", + "pub_false_positive_list = []\n", + "pub_false_negative_list = []\n", + "pub_calculation_methods = []\n", + "#calculation_methods.append( CALCULATION_METHOD_DEFAULT )\n", + "pub_calculation_methods.append( CALCULATION_METHOD_BINARY )\n", + "#calculation_methods.append( CACLULATION_METHOD_MACRO )\n", + "#calculation_methods.append( CALCULATION_METHOD_MICRO )\n", + "#calculation_methods.append( CALCULATION_METHOD_WEIGHTED )\n", + "pub_output_dictionary = None\n", + "pub_confusion_matrix = None\n", + "pub_false_positive_count = None\n", + "pub_false_negative_count = None\n", + "output_string = None\n", + "\n", + "# declare variables - precision/recall/F1\n", + "pub_method_to_result_map = None\n", + "pub_evaluation_tuple = None\n", + "pub_precision = None\n", + "pub_recall = None\n", + "pub_F1 = None\n", + "pub_precision_list = []\n", + "pub_recall_list = []\n", + "pub_F1_list = []\n", + "\n", + "# loop over publications\n", + "for publication_id in six.iterkeys( publication_to_lists_map ):\n", + " \n", + " if ( debug_flag == True ):\n", + " print( \"Publication ID: {}\".format( publication_id ) )\n", + " #-- END debug --#\n", + " \n", + " # get lists\n", + " pub_list_dictionary = publication_to_lists_map.get( publication_id, None )\n", + " pub_baseline_list = pub_list_dictionary.get( coding_evaluator.LIST_TYPE_BASELINE, None )\n", + " pub_derived_binary_list = pub_list_dictionary.get( coding_evaluator.LIST_TYPE_DERIVED_BINARY, None )\n", + " pub_derived_raw_list = pub_list_dictionary.get( coding_evaluator.LIST_TYPE_DERIVED_RAW, None )\n", + " pub_publication_id_per_citation_list = pub_list_dictionary.get( coding_evaluator.LIST_TYPE_PUBLICATION_ID, None )\n", + " pub_data_set_id_per_citation_list = pub_list_dictionary.get( coding_evaluator.LIST_TYPE_DATA_SET_ID, None )\n", + " \n", + " if ( debug_flag == True ):\n", + " # print lists:\n", + " print( \"====> baseline......: {}\".format( pub_baseline_list ) )\n", + " print( \"====> derived_binary: {}\".format( pub_derived_binary_list ) )\n", + " print( \"====> derived_raw...: {}\".format( pub_derived_raw_list ) )\n", + " print( \"====> publication_id: {}\".format( pub_publication_id_per_citation_list ) )\n", + " print( \"====> data_set_id...: {}\".format( pub_data_set_id_per_citation_list ) )\n", + " #-- END debug --#\n", + " \n", + " # call the precision and recall function\n", + " pub_output_dictionary = precision_recall_f1( pub_baseline_list, pub_derived_binary_list, pub_calculation_methods, do_print_IN = pub_debug_flag )\n", + "\n", + " if ( debug_flag == True ):\n", + " print( \"----> pub output dictionary: {}\".format( pub_output_dictionary ) )\n", + " #-- END debug --#\n", + " \n", + " # get confusion matrix\n", + " pub_confusion_matrix = pub_output_dictionary.get( RETURN_CONFUSION_MATRIX, None )\n", + " if ( debug_flag == True ):\n", + " print( \"Confusion Matrix: {}\".format( pub_confusion_matrix ) )\n", + " #-- END debug --#\n", + " \n", + " \n", + " if ( pub_confusion_matrix is not None ):\n", + " \n", + " # try to get false positives (cm[ 0 ][ 1 ]). If exception, is 0.\n", + " try:\n", + " \n", + " pub_false_positive_count = pub_confusion_matrix[ 0 ][ 1 ]\n", + "\n", + " if ( debug_flag == True ):\n", + " print( \"found FP!\" )\n", + " #-- END debug --#\n", + " \n", + " except:\n", + "\n", + " if ( debug_flag == True ):\n", + " print( \"no FP!\" )\n", + " #-- END debug --#\n", + " \n", + " # index doesn't exist. Set to 0.\n", + " pub_false_positive_count = 0\n", + " \n", + " #-- END try...except. --#\n", + " \n", + " # try to get false negatives (cm[ 1 ][ 0 ]). If exception, is 0.\n", + " try:\n", + " \n", + " pub_false_negative_count = pub_confusion_matrix[ 1 ][ 0 ]\n", + "\n", + " if ( debug_flag == True ):\n", + " print( \"found FN!\" )\n", + " #-- END debug --#\n", + " \n", + " except:\n", + "\n", + " if ( debug_flag == True ):\n", + " print( \"no FN!\" )\n", + " #-- END debug --#\n", + " \n", + " # index doesn't exist. Set to 0.\n", + " pub_false_negative_count = 0\n", + " \n", + " #-- END try...except. --#\n", + "\n", + " # add id and count to list.\n", + " pub_publication_id_list.append( publication_id )\n", + " pub_false_positive_list.append( pub_false_positive_count )\n", + " pub_false_negative_list.append( pub_false_negative_count )\n", + " \n", + " else:\n", + " \n", + " # no confusion matrix\n", + " print( \"ERROR - no confusion matrix!\" )\n", + " \n", + " #-- END check to see if confusion matrix --#\n", + " \n", + " # get results...\n", + " pub_precision = -1\n", + " pub_recall = -1\n", + " pub_F1 = -1\n", + " pub_method_to_result_map = pub_output_dictionary.get( RETURN_METHOD_TO_RESULT_MAP, None )\n", + " if ( pub_method_to_result_map is not None ):\n", + " \n", + " # get results... for binary calculation method\n", + " pub_evaluation_tuple = pub_method_to_result_map.get( CALCULATION_METHOD_BINARY, None )\n", + " if ( pub_evaluation_tuple is not None ):\n", + " \n", + " # get results\n", + " pub_precision = pub_evaluation_tuple[ 0 ]\n", + " pub_recall = pub_evaluation_tuple[ 1 ]\n", + " pub_F1 = pub_evaluation_tuple[ 2 ]\n", + " \n", + " else:\n", + "\n", + " # no results for binary calculation method\n", + " print( \"ERROR - no results for binary calculation method!\" )\n", + "\n", + " #-- END check to see if confusion matrix --#\n", + "\n", + " else:\n", + " \n", + " # no results for binary calculation method\n", + " print( \"ERROR - no results!\" )\n", + " \n", + " #-- END check to see if confusion matrix --#\n", + " \n", + " # add to lists\n", + " pub_precision_list.append( pub_precision )\n", + " pub_recall_list.append( pub_recall )\n", + " pub_F1_list.append( pub_F1 )\n", + " \n", + "#-- END loop over per-publication lists --#\n", + "\n", + "if ( debug_flag == True ):\n", + " print( pub_publication_id_list )\n", + " print( pub_false_positive_list )\n", + " print( pub_false_negative_list )\n", + "#-- END debug --#\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### false positives (FP)\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# summarize\n", + "output_string = \"\\n========================================\\nFalse Positives (FP):\"\n", + "print( output_string )\n", + "line_list.append( output_string )\n", + "\n", + "# declare variables\n", + "pub_false_positive_array = None\n", + "pub_false_positive_mean = None\n", + "pub_fp_pub_id_list = []\n", + "pub_fp_count_list = []\n", + "item_index = None\n", + "current_count = None\n", + "output_string = None\n", + "zipped_fp_lists = None\n", + "fp_row = None\n", + "\n", + "# convert false positive list to a numpy array and get the mean\n", + "pub_false_positive_array = numpy.array( pub_false_positive_list )\n", + "pub_false_positive_mean = numpy.mean( pub_false_positive_array )\n", + "\n", + "# loop over items, flag any that are over mean\n", + "item_index = -1\n", + "for current_count in pub_false_positive_list:\n", + " \n", + " # increment index\n", + " item_index += 1\n", + "\n", + " # get publication ID\n", + " publication_id = pub_publication_id_list[ item_index ]\n", + " \n", + " # is count greater than mean?\n", + " if ( current_count > pub_false_positive_mean ):\n", + " \n", + " # add to list\n", + " pub_fp_pub_id_list.append( publication_id )\n", + " pub_fp_count_list.append( current_count )\n", + "\n", + " else:\n", + " \n", + " if ( debug_flag == True ):\n", + " print( \"- pub {} FP {} <= mean ( {} )\".format( publication_id, current_count, pub_false_positive_mean ) )\n", + " #-- END debug --#\n", + " \n", + " #-- END loop over false positives. --#\n", + " \n", + "#-- END loop over publications --#\n", + "\n", + "# zip up the two lists (one list of pairs of values, rather than two lists).\n", + "\n", + "# ID order\n", + "zipped_fp_lists = list( zip( pub_fp_count_list, pub_fp_pub_id_list ) )\n", + "\n", + "# convert to ordered by count, then ID, largest to smallest.\n", + "zipped_fp_lists.sort( reverse = True )\n", + "\n", + "# anything in the list?\n", + "fp_count = len( zipped_fp_lists )\n", + "if( fp_count > 0 ):\n", + "\n", + " output_string = \"\\n==> {} False Positives (FP) above mean ( {} ):\".format( fp_count, pub_false_positive_mean )\n", + " print( output_string )\n", + " line_list.append( output_string )\n", + "\n", + " # output for review\n", + " for fp_row in zipped_fp_lists:\n", + "\n", + " # summarize\n", + " output_string = \"- pub {} FP {} > mean ( {} )\".format( fp_row[ 1 ], fp_row[ 0 ], pub_false_positive_mean )\n", + " print( output_string )\n", + " line_list.append( output_string )\n", + "\n", + " #-- END loop over items --#\n", + "\n", + "#-- END check to see if anything in list. --#\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### false negatives (FN)\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# summarize\n", + "output_string = \"\\n========================================\\nFalse Negatives (FN):\"\n", + "print( output_string )\n", + "line_list.append( output_string )\n", + "\n", + "# declare variables\n", + "pub_false_negative_array = None\n", + "pub_false_negative_mean = None\n", + "pub_fn_pub_id_list = []\n", + "pub_fn_count_list = []\n", + "item_index = None\n", + "current_count = None\n", + "output_string = None\n", + "zipped_fn_lists = None\n", + "fn_row = None\n", + "\n", + "# convert false negative list to a numpy array and get the mean\n", + "pub_false_negative_array = numpy.array( pub_false_negative_list )\n", + "pub_false_negative_mean = numpy.mean( pub_false_negative_array )\n", + "\n", + "# loop over items, flag any that are over mean\n", + "item_index = -1\n", + "for current_count in pub_false_negative_list:\n", + " \n", + " # increment index\n", + " item_index += 1\n", + "\n", + " # get publication ID\n", + " publication_id = pub_publication_id_list[ item_index ]\n", + " \n", + " # is count greater than mean?\n", + " if ( current_count > pub_false_negative_mean ):\n", + " \n", + " # add to list\n", + " pub_fn_pub_id_list.append( publication_id )\n", + " pub_fn_count_list.append( current_count )\n", + "\n", + " else:\n", + " \n", + " if ( debug_flag == True ):\n", + " print( \"- pub {} FN {} <= mean ( {} )\".format( publication_id, current_count, pub_false_negative_mean ) )\n", + " #-- END debug --#\n", + " \n", + " #-- END loop over false negatives. --#\n", + " \n", + "#-- END loop over publications --#\n", + "\n", + "# zip up the two lists (one list of pairs of values, rather than two lists).\n", + "\n", + "# ID order\n", + "zipped_fn_lists = list( zip( pub_fn_count_list, pub_fn_pub_id_list ) )\n", + "\n", + "# convert to ordered by count, then ID, largest to smallest.\n", + "zipped_fn_lists.sort( reverse = True )\n", + "\n", + "# anything in the list?\n", + "fn_count = len( zipped_fn_lists )\n", + "if( fn_count > 0 ):\n", + "\n", + " output_string = \"\\n==> {} False Negatives (FN) above mean ( {} ):\".format( fn_count, pub_false_negative_mean )\n", + " print( output_string )\n", + " line_list.append( output_string )\n", + "\n", + " # output for review\n", + " for fn_row in zipped_fn_lists:\n", + "\n", + " # summarize\n", + " output_string = \"- pub {} FN {} > mean ( {} )\".format( fn_row[ 1 ], fn_row[ 0 ], pub_false_negative_mean )\n", + " print( output_string )\n", + " line_list.append( output_string )\n", + "\n", + " #-- END loop over items --#\n", + " \n", + "#-- END check to see if anything in list. --#\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### output all publication-citation pairs\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# output all the full lists\n", + "current_index = -1\n", + "for item in publication_id_per_citation_list:\n", + " \n", + " # increment index\n", + " current_index += 1\n", + " \n", + " # get current values.\n", + " baseline_value = baseline_list[ current_index ]\n", + " derived_raw_value = derived_raw_list[ current_index ]\n", + " derived_binary_value = derived_binary_list[ current_index ]\n", + " pub_id_value = publication_id_per_citation_list[ current_index ]\n", + " data_set_id_value = data_set_id_per_citation_list[ current_index ]\n", + " \n", + " print( \"{}: pub ID {} - data set ID {} - baseline {} - binary {} - raw {}\".format( current_index, pub_id_value, data_set_id_value, baseline_value, derived_binary_value, derived_raw_value ) )\n", + " \n", + "#-- END loop over full lists. --#" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## graph precision and recall at n\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# set new precision_recall_graph_path\n", + "precision_recall_graph_path = \"{}/{}precision_recall_graph-cited{}.pdf\".format( output_folder_path, file_name_prefix, file_name_suffix )\n", + "\n", + "# declare variables\n", + "plot_details = None\n", + "\n", + "# output to file?\n", + "if ( output_to_file == True ):\n", + " \n", + " # output figure to file\n", + " plot_details = plot_precision_recall_n( baseline_list, derived_raw_list, \"evaluation\", output_path_IN = precision_recall_graph_path )\n", + "\n", + "else:\n", + " \n", + " # just output to standard out (as is possible)\n", + " plot_details = plot_precision_recall_n( baseline_list, derived_raw_list, \"evaluation\" )\n", + "\n", + "#-- END check to see if output graph to file --#\n", + "\n", + "# DEBUG?\n", + "if ( debug_flag == True ):\n", + "\n", + " # summarize\n", + " output_string = \"- plot details: {}\".format( plot_details )\n", + " print( output_string )\n", + " #line_list.append( output_string )\n", + "\n", + "#-- END DEBUG --#" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## output results to file\n", + "\n", + "- Back to [Table of Contents](#Table-of-Contents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# set results file path:\n", + "results_file_path = \"{}/{}evaluation_results-cited{}.txt\".format( output_folder_path, file_name_prefix, file_name_suffix )\n", + "\n", "# declare variables\n", "results_file = None\n", "line_list_string = None\n",