From 2e36c15a2df74a1e2ce3d4c18b5133c835ddebde Mon Sep 17 00:00:00 2001 From: "M. Zain Sohail" Date: Wed, 19 Jun 2024 15:40:12 +0200 Subject: [PATCH 1/4] update hextof notebook --- tutorial/4_hextof_workflow.ipynb | 97 ++++++++++++-------------------- 1 file changed, 37 insertions(+), 60 deletions(-) diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index 5cb79585..6d124866 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -50,16 +50,29 @@ "### Download and unpack data" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory." + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# data_path can be defined and used to store the data in a specific location\n", - "dataset.get(\"Gd_W110\") # Put in Path to a storage of at least 10 Gbyte free space.\n", - "path = dataset.dir\n", - "print(path)" + "cal_file = \"hextof_tutorial_cal.yaml\" # file to store the calibration parameters in\n", + "beamtime_dir = \"/asap3/flash/gpfs/pg2/2023/data/11019101\"\n", + "if os.path.exists(beamtime_dir) and os.access(beamtime_dir, os.R_OK):\n", + " path = beamtime_dir + \"/raw/hdf/offline/fl1user3\"\n", + " buffer_path = beamtime_dir + \"/processed/tutorial/\"\n", + "else:\n", + " # data_path can be defined and used to store the data in a specific location\n", + " dataset.get(\"Gd_W110\") # Put in Path to a storage of at least 10 Gbyte free space.\n", + " path = dataset.dir\n", + " buffer_path = path + \"/processed/\"" ] }, { @@ -99,58 +112,13 @@ "config_override = {\n", " \"core\": {\n", " \"paths\": {\n", - " \"data_raw_dir\": \"\",\n", - " \"data_parquet_dir\": \"\"\n", + " \"data_raw_dir\": path,\n", + " \"data_parquet_dir\": buffer_path\n", " },\n", " },\n", "}" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "config_override['core']['paths']['data_raw_dir'] = \"/asap3/flash/gpfs/pg2/2023/data/11019101/raw/hdf/offline/fl1user3\"\n", - "# If this will not work for you, please change it to a path where you have write access\n", - "config_override['core']['paths']['data_parquet_dir'] = \"/asap3/flash/gpfs/pg2/2023/data/11019101/processed\"\n", - "# So we write to user space\n", - "config_override['core']['paths']['data_parquet_dir'] = path + \"/processed\"\n", - "# If you aren't using maxwell and downloaded the data, use this path\n", - "config_override['core']['paths']['data_raw_dir'] = path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### cleanup previous config files\n", - "In this notebook, we will show how calibration parameters can be generated. Therefore we want to clean the local directory of previously generated files.\n", - "\n", - "**WARNING** running the cell below will delete the \"sed_config.yaml\" file in the local directory. If these contain precious calibration parameters, **DO NOT RUN THIS CELL**." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "local_folder_config = Path('./sed_config.yaml')\n", - "if local_folder_config.exists():\n", - " os.remove(local_folder_config)\n", - " print(f'deleted local config file {local_folder_config}')\n", - "assert not local_folder_config.exists()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -175,7 +143,7 @@ "metadata": {}, "outputs": [], "source": [ - "sp = SedProcessor(runs=[44762], config=config_override, system_config=config_file, collect_metadata=False, force_recreate=True)\n", + "sp = SedProcessor(runs=[44762], config=config_override, system_config=config_file, collect_metadata=False)\n", "# You can set collect_metadata=True if the scicat_url and scicat_token are defined" ] }, @@ -330,7 +298,7 @@ "metadata": {}, "outputs": [], "source": [ - "fig,ax = plt.subplots(1,3,figsize=(8,2), layout='tight')\n", + "fig,ax = plt.subplots(1,3,figsize=(10,3), layout='tight')\n", "res_chessy.plot(ax=ax[0], robust=True)\n", "res_t05.plot(ax=ax[1], robust=True)\n", "res_t10.plot(ax=ax[2], robust=True)" @@ -648,8 +616,8 @@ "metadata": {}, "outputs": [], "source": [ - "sp.save_energy_calibration()\n", - "sp.save_energy_offset()" + "sp.save_energy_calibration(cal_file)\n", + "sp.save_energy_offset(cal_file)" ] }, { @@ -665,7 +633,7 @@ "metadata": {}, "outputs": [], "source": [ - "sp.save_workflow_params()" + "sp.save_workflow_params(cal_file)" ] }, { @@ -692,6 +660,7 @@ "sp = SedProcessor(\n", " runs=[44824,44825,44826,44827],\n", " config=config_override,\n", + " user_config=cal_file,\n", " system_config=config_file,\n", " collect_metadata=False,\n", ")" @@ -767,7 +736,7 @@ " constant=-1463.7, # this is time zero\n", " flip_delay_axis=True, # invert the direction of the delay axis\n", " columns=['bam'], # use the bam to offset the values\n", - " weights=[0.001], # bam is in fs, delay in ps\n", + " weights=[-0.001], # bam is in fs, delay in ps\n", " preserve_mean=True # preserve the mean of the delay axis\n", ")" ] @@ -860,7 +829,7 @@ }, "outputs": [], "source": [ - "sp.save_delay_offsets()" + "sp.save_delay_offsets(cal_file)" ] }, { @@ -884,6 +853,7 @@ "sp = SedProcessor(\n", " runs=[44824,44825,44826,44827],\n", " config=config_override,\n", + " user_config=cal_file,\n", " system_config=config_file,\n", " collect_metadata=False\n", ")" @@ -964,6 +934,13 @@ "source": [ "sp.save('binned.tiff')" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -982,9 +959,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.8.12" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From 23ef8d0c4dd45e38087d28e2d6186acb09a5ac2c Mon Sep 17 00:00:00 2001 From: "M. Zain Sohail" Date: Wed, 19 Jun 2024 15:44:52 +0200 Subject: [PATCH 2/4] update hextof notebook --- tutorial/4_hextof_workflow.ipynb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index 6d124866..5c1c3106 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -47,14 +47,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Download and unpack data" + "### Get data paths" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory." + "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory.\n", + "\n", + "The paths are such that if you are on Maxwell, it uses those. Otherwise data is downloaded in current directory from Zenodo." ] }, { @@ -63,8 +65,7 @@ "metadata": {}, "outputs": [], "source": [ - "cal_file = \"hextof_tutorial_cal.yaml\" # file to store the calibration parameters in\n", - "beamtime_dir = \"/asap3/flash/gpfs/pg2/2023/data/11019101\"\n", + "beamtime_dir = \"/asap3/flash/gpfs/pg2/2023/data/11019101\" # on Maxwell\n", "if os.path.exists(beamtime_dir) and os.access(beamtime_dir, os.R_OK):\n", " path = beamtime_dir + \"/raw/hdf/offline/fl1user3\"\n", " buffer_path = beamtime_dir + \"/processed/tutorial/\"\n", @@ -97,6 +98,8 @@ }, "outputs": [], "source": [ + "# file to store the calibration parameters in\n", + "cal_file = \"hextof_tutorial_cal.yaml\" \n", "# pick the default configuration file for hextof@FLASH\n", "config_file = Path('../sed/config/flash_example_config.yaml')\n", "assert config_file.exists()" @@ -934,13 +937,6 @@ "source": [ "sp.save('binned.tiff')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 9a69e4dbf92d4be5e7848f6db53111bf44902454 Mon Sep 17 00:00:00 2001 From: "M. Zain Sohail" Date: Thu, 20 Jun 2024 16:58:46 +0200 Subject: [PATCH 3/4] make buffer path local to user --- tutorial/4_hextof_workflow.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index 5c1c3106..dd2cb1e6 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -68,7 +68,7 @@ "beamtime_dir = \"/asap3/flash/gpfs/pg2/2023/data/11019101\" # on Maxwell\n", "if os.path.exists(beamtime_dir) and os.access(beamtime_dir, os.R_OK):\n", " path = beamtime_dir + \"/raw/hdf/offline/fl1user3\"\n", - " buffer_path = beamtime_dir + \"/processed/tutorial/\"\n", + " buffer_path = \"Gd_W110/processed/\"\n", "else:\n", " # data_path can be defined and used to store the data in a specific location\n", " dataset.get(\"Gd_W110\") # Put in Path to a storage of at least 10 Gbyte free space.\n", @@ -941,7 +941,7 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3.9", "language": "python", "name": "python3" }, @@ -955,7 +955,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.16" } }, "nbformat": 4, From f30870d40bcdf763608bfcf074926654333f1da7 Mon Sep 17 00:00:00 2001 From: "M. Zain Sohail" Date: Mon, 8 Jul 2024 16:10:37 +0200 Subject: [PATCH 4/4] remove the cal file --- tutorial/4_hextof_workflow.ipynb | 90 ++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index dd2cb1e6..96936c4d 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -54,9 +54,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory.\n", + "The paths are such that if you are on Maxwell, it uses those. Otherwise data is downloaded in current directory from Zenodo.\n", "\n", - "The paths are such that if you are on Maxwell, it uses those. Otherwise data is downloaded in current directory from Zenodo." + "Generally, if it is your beamtime, you can both read the raw data and write to processed directory. However, for the public data, you can not write to processed directory." ] }, { @@ -98,8 +98,6 @@ }, "outputs": [], "source": [ - "# file to store the calibration parameters in\n", - "cal_file = \"hextof_tutorial_cal.yaml\" \n", "# pick the default configuration file for hextof@FLASH\n", "config_file = Path('../sed/config/flash_example_config.yaml')\n", "assert config_file.exists()" @@ -115,13 +113,58 @@ "config_override = {\n", " \"core\": {\n", " \"paths\": {\n", - " \"data_raw_dir\": path,\n", - " \"data_parquet_dir\": buffer_path\n", + " \"data_raw_dir\": \"\",\n", + " \"data_parquet_dir\": \"\"\n", " },\n", " },\n", "}" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config_override['core']['paths']['data_raw_dir'] = \"/asap3/flash/gpfs/pg2/2023/data/11019101/raw/hdf/offline/fl1user3\"\n", + "# If this will not work for you, please change it to a path where you have write access\n", + "config_override['core']['paths']['data_parquet_dir'] = \"/asap3/flash/gpfs/pg2/2023/data/11019101/processed\"\n", + "# So we write to user space\n", + "config_override['core']['paths']['data_parquet_dir'] = path + \"/processed\"\n", + "# If you aren't using maxwell and downloaded the data, use this path\n", + "config_override['core']['paths']['data_raw_dir'] = path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### cleanup previous config files\n", + "In this notebook, we will show how calibration parameters can be generated. Therefore we want to clean the local directory of previously generated files.\n", + "\n", + "**WARNING** running the cell below will delete the \"sed_config.yaml\" file in the local directory. If these contain precious calibration parameters, **DO NOT RUN THIS CELL**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_folder_config = Path('./sed_config.yaml')\n", + "if local_folder_config.exists():\n", + " os.remove(local_folder_config)\n", + " print(f'deleted local config file {local_folder_config}')\n", + "assert not local_folder_config.exists()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -146,7 +189,7 @@ "metadata": {}, "outputs": [], "source": [ - "sp = SedProcessor(runs=[44762], config=config_override, system_config=config_file, collect_metadata=False)\n", + "sp = SedProcessor(runs=[44762], config=config_override, system_config=config_file, collect_metadata=False, force_recreate=True)\n", "# You can set collect_metadata=True if the scicat_url and scicat_token are defined" ] }, @@ -218,7 +261,7 @@ "metadata": {}, "source": [ "### Binning\n", - "Here we define the parameters for binning the dataframe to an n-dimensional histogram, which we can then plot, analyse or save.\n", + "Here we define the parameters for binning the dataframe to an n-dimensional histogram, which we can then plot, analyze or save.\n", "\n", "If you never saw this before, the type after `:` is a \"hint\" to what type the object to the left will have. We include them here to make sure you know what each variable should be.\n", "```python\n", @@ -301,7 +344,7 @@ "metadata": {}, "outputs": [], "source": [ - "fig,ax = plt.subplots(1,3,figsize=(10,3), layout='tight')\n", + "fig,ax = plt.subplots(1,3,figsize=(8,2), layout='tight')\n", "res_chessy.plot(ax=ax[0], robust=True)\n", "res_t05.plot(ax=ax[1], robust=True)\n", "res_t10.plot(ax=ax[2], robust=True)" @@ -388,7 +431,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now, to determine propper binning ranges, let's have again a look at the event histograms:" + "Now, to determine proper binning ranges, let's have again a look at the event histograms:" ] }, { @@ -426,7 +469,7 @@ "outputs": [], "source": [ "plt.figure()\n", - "res.plot.line(x='dldTime'); # the ; here is to suppres an annoying output" + "res.plot.line(x='dldTime'); # the ; here is to suppress an annoying output" ] }, { @@ -556,7 +599,7 @@ "metadata": {}, "source": [ "### correct offsets\n", - "The energy axis is now correct, but still the curves do not stack on eachother as we are not compensating for the `sampleBias`. In the same way, we can compensate the photon energy (`monocrhomatorPhotonEnergy`) and the `tofVoltage` " + "The energy axis is now correct, but still the curves do not stack on each other as we are not compensating for the `sampleBias`. In the same way, we can compensate the photon energy (`monochromatorPhotonEnergy`) and the `tofVoltage` " ] }, { @@ -610,7 +653,7 @@ "metadata": {}, "source": [ "### save the calibration parameters\n", - "The parameters we have found can be saved to a file, so that we can use them later. This means the calibrtion can be used for different runs." + "The parameters we have found can be saved to a file, so that we can use them later. This means the calibration can be used for different runs." ] }, { @@ -619,15 +662,15 @@ "metadata": {}, "outputs": [], "source": [ - "sp.save_energy_calibration(cal_file)\n", - "sp.save_energy_offset(cal_file)" + "sp.save_energy_calibration()\n", + "sp.save_energy_offset()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "A more general function, which saves parameters for all the calibrations performed. Use either the above or below function. They are equivalent (and overwrite eachother)" + "A more general function, which saves parameters for all the calibrations performed. Use either the above or below function. They are equivalent (and overwrite each other)" ] }, { @@ -636,7 +679,7 @@ "metadata": {}, "outputs": [], "source": [ - "sp.save_workflow_params(cal_file)" + "sp.save_workflow_params()" ] }, { @@ -663,7 +706,6 @@ "sp = SedProcessor(\n", " runs=[44824,44825,44826,44827],\n", " config=config_override,\n", - " user_config=cal_file,\n", " system_config=config_file,\n", " collect_metadata=False,\n", ")" @@ -798,7 +840,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You may note some intensity variation along the delay axis. This comes mainly from inhomogenous speed of the delay stage, and thus inquivalently time spend on every delay point. This can be corrected for by normalizing the data to the aquisition time per delay point:" + "You may note some intensity variation along the delay axis. This comes mainly from inhomogeneous speed of the delay stage, and thus inequivalent amounts of time spent on every delay point. This can be corrected for by normalizing the data to the acquisition time per delay point:" ] }, { @@ -832,7 +874,7 @@ }, "outputs": [], "source": [ - "sp.save_delay_offsets(cal_file)" + "sp.save_delay_offsets()" ] }, { @@ -856,7 +898,6 @@ "sp = SedProcessor(\n", " runs=[44824,44825,44826,44827],\n", " config=config_override,\n", - " user_config=cal_file,\n", " system_config=config_file,\n", " collect_metadata=False\n", ")" @@ -956,6 +997,13 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.16" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {}, + "version_major": 2, + "version_minor": 0 + } } }, "nbformat": 4,