{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Test notebook for JWebbinars\n", "## Running the pipeline on NIRISS simulated data\n", "\n", "Test environment for the JWebbinars with just\n", "- jwst==0.17.1\n", "- jdaviz\n", "\n", "Pinned CRDS version - 0645" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%env CRDS_CONTEXT=jwst_0645.pmap\n", "\n", "#system general\n", "import os\n", "import fnmatch\n", "import sys\n", "import urllib.request\n", "\n", "#plotting and tabling\n", "import matplotlib.pyplot as plt\n", "import matplotlib.patches as patches\n", "import astropy\n", "from astropy.io import fits\n", "from astropy.io import ascii\n", "import astropy.wcs as pywcs\n", "import numpy as np\n", "import h5py\n", "import pandas as pd\n", "\n", "#pipeline\n", "import jwst\n", "from jwst.pipeline import Detector1Pipeline\n", "from jwst.pipeline import Image2Pipeline\n", "from jwst.pipeline import Image3Pipeline\n", "from jwst.pipeline import Spec2Pipeline\n", "from jwst.pipeline import Spec3Pipeline\n", "#wcs\n", "from jwst import assign_wcs\n", "from jwst.assign_wcs import assign_wcs_step\n", "#datamodels\n", "from jwst.datamodels import image, DrizProductModel, WavelengthrangeModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print('Versions')\n", "print('astropy:',astropy.__version__)\n", "print('pipeline:',jwst.__version__)\n", "print('numpy:',np.__version__)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#define a plotting funciton for the mirage images\n", "def show(array,title,min=0,max=1000):\n", " plt.figure(figsize=(12,12))\n", " plt.imshow(array,clim=(min,max))\n", " plt.title(title)\n", " plt.colorbar().set_label('DN/s')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## File names" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#starting files\n", "uncalfile = \"https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/jwebbinar_test/basic_F150W_direct_uncal.fits\"\n", "uncalspecfileC = \"https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/jwebbinar_test/basic_F150W_WFSSC_uncal.fits\"\n", "uncalspecfileR = \"https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/jwebbinar_test/basic_F150W_WFSSR_uncal.fits\"\n", "\n", "#the json files need to be downloaded\n", "associationClink = \"https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/jwebbinar_test/basic_F150W_WFSSC_stage2.json\"\n", "associationC = \"basic_F150W_WFSSC_stage2.json\"\n", "associationRlink = \"https://data.science.stsci.edu/redirect/JWST/jwst-data_analysis_tools/jwebbinar_test/basic_F150W_WFSSR_stage2.json\"\n", "associationR = \"basic_F150W_WFSSR_stage2.json\"\n", "urllib.request.urlretrieve(associationClink, associationC)\n", "urllib.request.urlretrieve(associationRlink, associationR)\n", "\n", "#files that will be produced\n", "ratefile = \"./basic_F150W_direct_rate.fits\"\n", "calfile = \"./basic_F150W_direct_cal.fits\"\n", "catfile = \"./basic_F150W_direct_cat.ecsv\"\n", "ratespecfileC = \"./basic_F150W_WFSSC_rate.fits\"\n", "calspecfileC = \"basic_F150W_WFSSC_cal.fits\"\n", "x1dspecfileC = \"./basic_F150W_WFSSC_x1d.fits\"\n", "ratespecfileR = \"./basic_F150W_WFSSR_rate.fits\"\n", "calspecfileR = \"./basic_F150W_WFSSR_cal.fits\"\n", "x1dspecfileR = \"./basic_F150W_WFSSR_x1d.fits\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run pipeline" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#check original header\n", "uncalimaged = fits.open(uncalfile)\n", "print(uncalimaged.info())\n", "uncalimaged[0].header" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run pipeline stage1, 2, and 3 on uncal direct image ##\n", "\n", "**Config files are saved locally**" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage1 = Detector1Pipeline.call(uncalfile, config_file='./configfiles/calwebb_detector1.cfg', save_results=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#plot rate image\n", "rateimage = fits.open(ratefile)\n", "\n", "print(rateimage.info())\n", "\n", "plt.figure(figsize=(12,12))\n", "plt.imshow((rateimage[1].data),clim=(-1,2))\n", "plt.xlim(1000,1100)\n", "plt.ylim(1000,1100)\n", "plt.title(\"Science image\")\n", "plt.colorbar()\n", "\n", "plt.figure(figsize=(12,12))\n", "plt.imshow((rateimage[3].data),clim=(0,10))\n", "plt.xlim(1000,1100)\n", "plt.ylim(1000,1100)\n", "plt.title(\"DQ array\")\n", "plt.colorbar()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage2 = Image2Pipeline.call(ratefile, config_file='./configfiles/calwebb_image2.cfg', save_results=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage3 = Image3Pipeline.call(calfile, config_file='./configfiles/calwebb_image3.cfg', save_results=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check catalog ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#plot the sources obtained\n", "cat = pd.read_table(catfile,comment='#',sep='\\s+')\n", "cat.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Plot sources on cal images." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#plot cal image and detected sources\n", "calimage = fits.open(calfile)\n", "plt.figure(figsize=(12,12))\n", "plt.imshow((calimage[1].data),clim=(0,2))\n", "plt.xlim(0,2048)\n", "plt.ylim(0,2048)\n", "plt.colorbar()\n", "\n", "plt.scatter(cat['xcentroid'],cat['ycentroid'],s=80, facecolors='none', edgecolors='r')\n", "for i in range (0,len(cat)):\n", " plt.annotate(cat.loc[i,'id'], (cat.loc[i,'xcentroid'], cat.loc[i,'ycentroid']),color='white')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check headers and prepare for pipeline. Grism C ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#check original header\n", "uncalimages = fits.open(uncalspecfileC)\n", "print(uncalimages.info())\n", "uncalimages[0].header" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run pipeline stage 1." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage1spec = Detector1Pipeline.call(uncalspecfileC, config_file='./configfiles/calwebb_detector1.cfg', save_results=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's look at the rate image" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#check the rate image\n", "ratespecimage = fits.open(ratespecfileC)\n", "ratespecimage.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ratespecimage[0].header" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#plot rate image\n", "ratespecimage = fits.open(ratespecfileC)\n", "plt.figure(figsize=(12,12))\n", "plt.imshow(ratespecimage[1].data,clim=(0,1))\n", "#plt.xlim(750,950)\n", "plt.ylim(0,2048)\n", "plt.colorbar()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Use the rate image and catalog to write the association file and run stage 2. ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#run stage 2 with an association (rate file + catalog)\n", "#use catalog with just one source\n", "print(ratespecfileC)\n", "print(catfile)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage2spec = Spec2Pipeline.call(associationC, config_file='./configfiles/calwebb_spec2.cfg', save_results=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's check the cal image" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "##see output file\n", "calspecimage = fits.open(calspecfileC)\n", "calspecimage.info()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Plot the extracted spectra ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "spec1 = calspecimage[1].data\n", "spec2 = calspecimage[8].data\n", "spec3 = calspecimage[15].data\n", "spec4 = calspecimage[22].data\n", "spec5 = calspecimage[29].data\n", "\n", "plt.figure(figsize=(20,20))\n", "\n", "plt.subplot(5,1,1)\n", "ys,xs = spec1.shape\n", "plt.imshow(spec1, extent=[0,xs,0,ys])\n", "plt.colorbar()\n", "plt.clim(0,1000)\n", "plt.subplot(5,1,2)\n", "ys,xs = spec2.shape\n", "plt.imshow(spec2, extent=[0,xs,0,ys])\n", "plt.subplot(5,1,3)\n", "ys,xs = spec3.shape\n", "plt.imshow(spec3, extent=[0,xs,0,ys])\n", "plt.subplot(5,1,4)\n", "ys,xs = spec4.shape\n", "plt.imshow(spec4, extent=[0,xs,0,ys])\n", "plt.subplot(5,1,5)\n", "ys,xs = spec5.shape\n", "plt.imshow(spec5, extent=[0,xs,0,ys])\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#plot one row of the 2D spectrum just to look at it\n", "#len(spec1[0])\n", "\n", "spectot = np.zeros(len(spec1[0]))\n", "for i in range (0,spec1.shape[0]):\n", " spectot = spectot + spec1[i]\n", " \n", "xax = np.arange(0,len(spec1[0]),1)\n", "plt.plot(xax,spectot)\n", "#plt.ylim(0,10000)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#the wavelength extension\n", "calspecimage[1].header" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check the boxes on the image going through the pipeline individual steps. ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#lets check the boxes on the image\n", "from jwst.extract_2d.extract_2d_step import Extract2dStep\n", "from jwst.lib.catalog_utils import SkyObject\n", "from jwst.assign_wcs import util\n", "from jwst.assign_wcs.util import create_grism_bbox\n", "\n", "grism_image = image.ImageModel(ratespecfileC)\n", "wcs_step = assign_wcs.AssignWcsStep()\n", "grism_with_wcs = wcs_step(grism_image)\n", "grism_with_wcs.meta.source_catalog = catfile\n", "\n", "e2d = Extract2dStep()\n", "ref_types = e2d.reference_file_types\n", "reference_file_names = {}\n", "for name in ref_types:\n", " reffile = e2d.get_reference_file(grism_with_wcs, name)\n", " reference_file_names[name] = reffile if reffile else \"\"\n", "\n", "required_fields = list(SkyObject()._fields)\n", "grism_objects = util.get_object_info(catfile)\n", "\n", "print(grism_objects[0])\n", "\n", "grism_objects = create_grism_bbox(grism_with_wcs, reference_file_names, mmag_extract=99.0, extract_orders=None)\n", "\n", "# visualize the images with some bounding boxes\n", "dispim=fits.getdata(ratespecfileC)\n", "ys,xs = dispim.shape\n", "fig2 = plt.figure(figsize=(4,4), dpi=150)\n", "\n", "# Plot the grism image\n", "ax2 = fig2.add_subplot(1, 1, 1)\n", "ax2.ticklabel_format(useOffset=False)\n", "ax2.set_title(ratespecfileC.split(\"/\")[-1], fontsize=8)\n", "\n", "# rectangle patches are xmin, ymin, width, height\n", "order=1\n", "tt = grism_objects #[test_boxes[18], test_boxes[12], test_boxes[8]]\n", "for obj in tt:\n", " try:\n", " y,x = obj.order_bounding[order]\n", " ymin, ymax = y\n", " xmin, xmax = x\n", " xcenter, ycenter = obj.xcentroid, obj.ycentroid\n", " ax2.add_patch(patches.Rectangle((xmin, ymin),\n", " xmax - xmin,\n", " ymax - ymin,\n", " fill=False, color='red'))\n", " ax2.add_patch(patches.Circle((xcenter, ycenter), 10, color='blue'))\n", " except KeyError:\n", " pass\n", "\n", "ax2.imshow(dispim, origin='lower', extent=[0,xs,0,ys], vmin=-1, vmax=1)\n", "#ax2.imshow(dispim, origin='lower', vmin=-1, vmax=1)\n", "#ax2.set_xlim(750,1000)\n", "#ax2.set_ylim(400,600)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### What's in x1d?" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "x1dspecfile = \"singlesource_f200w_april_nis_x1d.fits\"\n", "x1dspecimage = fits.open(x1dspecfile)\n", "x1dspecimage.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "x1dspecimage = fits.open(x1dspecfileC)\n", "x1dspecimage.info()\n", "\n", "spec1_1d = x1dspecimage[4].data\n", "#spec2_1d = x1dspecimage[2].data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.plot(spec1_1d['WAVELENGTH'],np.log10(spec1_1d['FLUX']))\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check headers and prepare for pipeline. Grism R ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#check original header\n", "uncalimages = fits.open(uncalspecfileR)\n", "print(uncalimages.info())\n", "uncalimages[0].header" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run pipeline stage 1." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage1spec = Detector1Pipeline.call(uncalspecfileR, config_file='./configfiles/calwebb_detector1.cfg', save_results=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#check the rate image\n", "ratespecimage = fits.open(ratespecfileR)\n", "ratespecimage.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ratespecimage[0].header" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [], "source": [ "#plot rate image\n", "ratespecimage = fits.open(ratespecfileR)\n", "plt.figure(figsize=(12,12))\n", "plt.imshow(ratespecimage[1].data,clim=(0,1))\n", "#plt.xlim(750,950)\n", "plt.ylim(0,2048)\n", "plt.colorbar()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run stage 2 with an association (rate file + catalog)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(ratespecfileR)\n", "print(catfile)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stage2spec = Spec2Pipeline.call(associationR, config_file='./configfiles/calwebb_spec2.cfg', save_results=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "##see output file\n", "calspecimage = fits.open(calspecfileR)\n", "calspecimage.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "spec1 = calspecimage[1].data\n", "spec2 = calspecimage[8].data\n", "spec3 = calspecimage[15].data\n", "spec4 = calspecimage[22].data\n", "spec5 = calspecimage[29].data\n", "\n", "plt.figure(figsize=(20,20))\n", "\n", "plt.subplot(5,1,1)\n", "ys,xs = spec1.shape\n", "plt.imshow(spec1, extent=[0,xs,0,ys])\n", "plt.colorbar()\n", "plt.clim(0,1000)\n", "plt.subplot(5,1,2)\n", "ys,xs = spec2.shape\n", "plt.imshow(spec2, extent=[0,xs,0,ys])\n", "plt.subplot(5,1,3)\n", "ys,xs = spec3.shape\n", "plt.imshow(spec3, extent=[0,xs,0,ys])\n", "plt.subplot(5,1,4)\n", "ys,xs = spec4.shape\n", "plt.imshow(spec4, extent=[0,xs,0,ys])\n", "plt.subplot(5,1,5)\n", "ys,xs = spec5.shape\n", "plt.imshow(spec5, extent=[0,xs,0,ys])\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#lets check the boxes on the image\n", "from jwst.extract_2d.extract_2d_step import Extract2dStep\n", "from jwst.lib.catalog_utils import SkyObject\n", "from jwst.assign_wcs import util\n", "from jwst.assign_wcs.util import create_grism_bbox\n", "\n", "grism_image = image.ImageModel(ratespecfileR)\n", "wcs_step = assign_wcs.AssignWcsStep()\n", "grism_with_wcs = wcs_step(grism_image)\n", "grism_with_wcs.meta.source_catalog = catfile\n", "\n", "e2d = Extract2dStep()\n", "ref_types = e2d.reference_file_types\n", "reference_file_names = {}\n", "for name in ref_types:\n", " reffile = e2d.get_reference_file(grism_with_wcs, name)\n", " reference_file_names[name] = reffile if reffile else \"\"\n", "\n", "required_fields = list(SkyObject()._fields)\n", "grism_objects = util.get_object_info(catfile)\n", "\n", "print(grism_objects[0])\n", "\n", "grism_objects = create_grism_bbox(grism_with_wcs, reference_file_names, mmag_extract=99.0, extract_orders=None)\n", "\n", "# visualize the images with some bounding boxes\n", "dispim=fits.getdata(ratespecfileR)\n", "ys,xs = dispim.shape\n", "fig2 = plt.figure(figsize=(4,4), dpi=150)\n", "\n", "# Plot the grism image\n", "ax2 = fig2.add_subplot(1, 1, 1)\n", "ax2.ticklabel_format(useOffset=False)\n", "ax2.set_title(ratespecfileR.split(\"/\")[-1], fontsize=8)\n", "\n", "# rectangle patches are xmin, ymin, width, height\n", "order=1\n", "tt = grism_objects #[test_boxes[18], test_boxes[12], test_boxes[8]]\n", "for obj in tt:\n", " try:\n", " y,x = obj.order_bounding[order]\n", " ymin, ymax = y\n", " xmin, xmax = x\n", " xcenter, ycenter = obj.xcentroid, obj.ycentroid\n", " ax2.add_patch(patches.Rectangle((xmin, ymin),\n", " xmax - xmin,\n", " ymax - ymin,\n", " fill=False, color='red'))\n", " ax2.add_patch(patches.Circle((xcenter, ycenter), 10, color='blue'))\n", " except KeyError:\n", " pass\n", "\n", "ax2.imshow(dispim, origin='lower', extent=[0,xs,0,ys], vmin=-1, vmax=1)\n", "#ax2.imshow(dispim, origin='lower', vmin=-1, vmax=1)\n", "#ax2.set_xlim(350,550)\n", "#ax2.set_ylim(300,440)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "x1dspecimage = fits.open(x1dspecfileR)\n", "x1dspecimage.info()\n", "\n", "spec1_1d = x1dspecimage[4].data\n", "spec2_1d = x1dspecimage[2].data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.plot(spec1_1d['WAVELENGTH'],np.log10(spec1_1d['FLUX']))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "x1dspecimage[1].header" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.1" } }, "nbformat": 4, "nbformat_minor": 2 }