{"type":"doc","content":[{"type":"paragraph","content":[{"text":"We are working on porting our python 2 PGE's to python 3. Individual PGE's will be tested and validated against equivalent datasets in C-cluster. Once all PGE's have been validated, a system level test of the pipeline will be performed to ensure all trigger rules are in place and all expected products are generated.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Pipeline","type":"text"}]},{"type":"paragraph","content":[{"text":"","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"PGE's to port to Python3","type":"text"}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"1"},"content":[{"text":"AOI Ops Report","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"2"},"content":[{"text":"AOI Enumerator Submitter","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"3"},"content":[{"text":"TopsApp","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"4"},"content":[{"text":"GUNW Completeness Evaluator","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"5"},"content":[{"text":"Blacklist Generation","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"6"},"content":[{"text":"Greylist Generation","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"7"},"content":[{"text":"AOI Based IPF Scraper","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"8"},"content":[{"text":"AOI Based Acquisition Scraper","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"9"},"content":[{"text":"Add Machine Tag","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"10"},"content":[{"text":"Add User Tag","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"11"},"content":[{"text":"Product Delivery","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"12"},"content":[{"text":"Create AOI","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"13"},"content":[{"text":"IPF ASF Scraper for Acq","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"14"},"content":[{"text":"IPF SciHub Scraper for Acq","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"15"},"content":[{"text":"Orbit Crawler","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"16"},"content":[{"text":"Orbit Ingest","type":"text"}]},{"type":"taskItem","attrs":{"state":"TODO","localId":"17"},"content":[{"text":"LAR","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"18"},"content":[{"text":"SLC sling","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"19"},"content":[{"text":"Enumerator","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"TODO","localId":"20"},"content":[{"text":"SAR avail","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"21"},"content":[{"text":"SLCP-COR","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"22"},"content":[{"text":"SLCP-PM","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"TODO","localId":"23"},"content":[{"text":"COD","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"24"},"content":[{"text":"Evaluator","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"DONE","localId":"25"},"content":[{"text":"Localizer","type":"text"}]}]},{"type":"taskList","attrs":{"localId":""},"content":[{"type":"taskItem","attrs":{"state":"TODO","localId":"26"},"content":[{"text":"multi acquisition localizer","type":"text"}]}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":2},"content":[{"text":"PGE I/O","type":"text"}]},{"type":"table","attrs":{"layout":"wide"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"PGE test list is incomplete. We are currently focusing on the main pipeline","type":"text"}]}]}]}]},{"type":"paragraph"},{"type":"expand","attrs":{"title":"PGE list"},"marks":[{"type":"breakout","attrs":{"mode":"wide"}}],"content":[{"type":"paragraph","content":[{"text":"AOI based submission of acq scraper jobs","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: develop","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"AOI based submission of ipf scraper jobs","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: develop","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"AOI Enumerator Submitter","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: develop","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"AOI Merged Track Stitcher","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: python3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"AOI Validate Acquisitions","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: develop","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"AWS Get Script","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: v0.0.8","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Data Sling Extract for asf","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: python3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Data Sling Extract for Scihub","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"build: python3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"input:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"output:","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Test Procedures","type":"text"}]},{"type":"expand","attrs":{"title":"Enumerator Submitter"},"marks":[{"type":"breakout","attrs":{"mode":"wide"}}],"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Run the enumerator submitter as an on-demand job to generate the acquisition lists. Compare output acquisition lists to those generated over the same AOI on C-cluster.","type":"text"}]},{"type":"paragraph","content":[{"text":"Github Repo: ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/aoi_enumerator_submitter"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"All expected acquisition lists were successfully generated","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (a single AOI)","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (acquisition lists)","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"Acquisition Enumerator"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Run the AOI track acquisition enumerator as an on-demand job to generate the acquisition lists. Compare output acquisition lists to those generated over the same AOI on C-cluster.","type":"text"}]},{"type":"paragraph","content":[{"text":"Github Repo: ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/standard_product.git"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"All expected acquisition lists were successfully generated","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (","type":"text"},{"text":"S1-AUX_POEORB","type":"text","marks":[{"type":"link","attrs":{"href":"https://c-datasets.aria.hysds.io/tosca/S1-AUX_POEORB"}}]},{"text":")","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (acquisition lists)","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"Localizer"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Run the localizer as an on-demand job to generate the sling jobs for the missing slc's.","type":"text"}]},{"type":"paragraph","content":[{"text":"Github Repo: ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/standard_product.git"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"All identified missing slc's from acquisition list input set have sling jobs spun up for them.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"There are no sling jobs generated for slc's already in the system.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (","type":"text"},{"text":"a","type":"text","marks":[{"type":"link","attrs":{"href":"https://c-datasets.aria.hysds.io/tosca/S1-AUX_POEORB"}}]},{"text":"cquisition lists)","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (sling jobs)","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"Evaluator"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Run the evaluator as an on-demand job to ensure all expected ifg-cfg's are generated.","type":"text"}]},{"type":"paragraph","content":[{"text":"Github Repo: ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/standard_product.git"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Compare ifg-cfg datasets that were generated to those that were generated on C-cluster over the same set of SLC's.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (slc's)","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (ifg-cfgs)","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"TopsApp"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Run the topsapp PGE as an on-demand job to generate GUNW's.","type":"text"}]},{"type":"paragraph","content":[{"text":"Github Repo: ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/ariamh.git"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"GUNW's for all input ifg-cfg's are generated and compared to those that were generated over the same ifg-cfg set.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (ifg-cfg)","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (s1-gunw)","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"Completeness Evaluator"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Run the completeness evaluator as an on-demand job to generate the aoi_track dataset if the track is complete.","type":"text"}]},{"type":"paragraph","content":[{"text":"Github Repo: ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/standard_product_completeness_evaluator.git"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If any GUNW is missing over a single date-pair spatial extent of aoi, aoi_track dataset is not generated.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If all GUNW's over a single date-pair spatial extent of aoi exist, aoi_track is generated.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (gunw)","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" (aoi track)","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"Sling/Extract"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Ensures this PGE pulls SLC products from ASF endpoint and ingests them into S3/ES when PGE is run as an on-demand job.","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Job must complete successfully","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"SLCs must be registered in S3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"SLCs must be found when faceting on them in tosca","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":"","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"expand","attrs":{"title":"Sling/Extract"},"content":[{"type":"paragraph","content":[{"text":"Test Description:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Ensures this PGE pulls SLC products from SciHub endpoint and ingests them into S3/ES when PGE is run as an on-demand job.","type":"text"}]},{"type":"paragraph","content":[{"text":"Pass Criteria:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Job must complete successfully","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"SLCs must be registered in S3","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"SLCs must be found when faceting on them in tosca","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Input dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Output dataset: ","type":"text","marks":[{"type":"strong"}]},{"text":"","type":"text"}]},{"type":"paragraph","content":[{"text":"Summary:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"

","type":"text"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Process","type":"text"}]},{"type":"paragraph","content":[{"text":"For this example we are porting the aoi ops report.","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a python3 virtual environment.","type":"text"},{"type":"hardBreak"},{"text":" ","type":"text"},{"text":"virtualenv env3 -p python3 to create the virtual environment","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Start the python3 virtual environment.","type":"text"},{"type":"hardBreak"},{"text":"source ~/env3/bin/activate to start a python 3 virtual environment","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Pull contents of repo on a new branch named ","type":"text"},{"text":"python3","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run futurize over the contents of the repo.","type":"text"},{"type":"hardBreak"},{"text":"pip install future","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"},{"text":"cd ","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"},{"text":"futurize -w -n -p .","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"},{"type":"hardBreak"},{"text":"Output will show what has been changed.","type":"text"},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ssh into e mozart to add the python3 converted pge using this command: ","type":"text"},{"type":"hardBreak"},{"text":"sds ci add_job -k -b develop ","type":"text","marks":[{"type":"em"}]},{"type":"inlineCard","attrs":{"url":"https://github.com/aria-jpl/standard_product_report.git"}},{"text":" s3","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Go to e Jenkins and click configure. Specify the branch as python3 branch and build. Check the docker file and change FROM to the latest branch. Wait for build to complete successfully. May take a few minutes. Will publish job to e cluster automatically.","type":"text"},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Go to e cluster and run job. step into container if you need to debug stuff.","type":"text"},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Once job runs successfully, push changes to dev.","type":"text"}]}]}]},{"type":"paragraph","content":[{"type":"hardBreak"}]},{"type":"paragraph","content":[{"type":"hardBreak"}]}],"version":1}

Browser not supported