From baa99df934025b80fd8fd6cd51b664a697d553b1 Mon Sep 17 00:00:00 2001 From: Michele Date: Fri, 14 Jan 2022 16:14:06 +0000 Subject: [PATCH] Updated notebooks and eo_utils.py --- SInCohMap_openEO_Sentinel2.ipynb | 15 +- SInCohMap_openEO_examples.ipynb | 256 ++++++++++++++++++++++++------- 2 files changed, 212 insertions(+), 59 deletions(-) diff --git a/SInCohMap_openEO_Sentinel2.ipynb b/SInCohMap_openEO_Sentinel2.ipynb index 828b63c..e132d74 100644 --- a/SInCohMap_openEO_Sentinel2.ipynb +++ b/SInCohMap_openEO_Sentinel2.ipynb @@ -32,7 +32,20 @@ "- Common use cases for optical data:\n", " - Create and apply a cloud mask\n", " - Create a cloud free median composite\n", - " - Mask out the clouds and compute the monthly mean NDVI" + " - Mask out the clouds and compute the monthly mean NDVI\n", + " \n", + "## FAQ:\n", + "**Q: I receive a 403 error, what does it mean?**\n", + "\n", + "A: If you get a 403 error, it usually means that the connection with the openEO back-end dropped. Please re run\n", + "\n", + "`conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")`\n", + "\n", + "and re run you code from `load_collection` onwards.\n", + "\n", + "**Q: I receive a 500 error, what does it mean?**\n", + "\n", + "A: It is a server error: something went wrong processing your request. Please check carefully that the area and time range you are requesting are available in the datacube (you can use `conn.describe_collection('COLLECTION_NAME')`) Currently the error logs are not passed if you run your request as a synchronous call (i.e. using `.download()`). If you run you process as a batch job you will get a more informative error message." ] }, { diff --git a/SInCohMap_openEO_examples.ipynb b/SInCohMap_openEO_examples.ipynb index 1c21e69..b37a9b6 100644 --- a/SInCohMap_openEO_examples.ipynb +++ b/SInCohMap_openEO_examples.ipynb @@ -25,16 +25,18 @@ "\n", "Getting started guide for openEO with python: https://openeo.org/documentation/1.0/python/\n", "\n", - "## Topics:\n", - "- Login to Eurac's openEO backend containing the SInCohMap datacubes\n", - "- Discover the data sets and processes\n", - "- Access and process data via openEO processes\n", - "- Common use cases for radar data:\n", - " - Compute the intensity over the area of interest with different Multi Look sizes in SAR and geocoded domain.\n", - " - Geocode the obtained values.\n", - " - Compute the coherence for a given temporal baseline.\n", - "- Common use cases for optical data:\n", - " - Mask out the clouds and compute the weekly mean NDVI" + "## FAQ:\n", + "**Q: I receive a 403 error, what does it mean?**\n", + "\n", + "A: If you get a 403 error, it usually means that the connection with the openEO back-end dropped. Please re run\n", + "\n", + "`conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")`\n", + "\n", + "and re run you code from `load_collection` onwards.\n", + "\n", + "**Q: I receive a 500 error, what does it mean?**\n", + "\n", + "A: It is a server error: something went wrong processing your request. Please check carefully that the area and time range you are requesting are available in the datacube (you can use `conn.describe_collection('COLLECTION_NAME')`) Currently the error logs are not passed if you run your request as a synchronous call (i.e. using `.download()`). If you run you process as a batch job you will get a more informative error message." ] }, { @@ -47,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "id": "0205484c-3eff-4825-bdae-78357b775897", "metadata": { "scrolled": true @@ -16033,7 +16035,7 @@ }, { "cell_type": "code", - "execution_count": 175, + "execution_count": 8, "id": "a4fa0ed5-6678-4ca7-9442-c8521cfa0c90", "metadata": { "scrolled": true @@ -24483,18 +24485,21 @@ "id": "a1a06e33-674d-4890-a3d9-be0d06c54ceb", "metadata": {}, "source": [ + "\n", "### Load the datacube" ] }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "id": "d0775a26-949c-4210-b737-27786a8490f1", "metadata": { "scrolled": true }, "outputs": [], "source": [ + "conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")\n", + "\n", "collection = 'SAR2Cube_SInCohMap_S1_L0_147_ASC_DONYANA'\n", "temporal_extent = [\"2018-06-01T00:00:00.000Z\", \"2018-06-30T00:00:00.000Z\"]\n", "\n", @@ -24697,24 +24702,6 @@ "Load the datacube" ] }, - { - "cell_type": "code", - "execution_count": 134, - "id": "912de89d-5e9a-4068-a75a-575a929f3529", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Authenticated using refresh token.\n" - ] - } - ], - "source": [ - "conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")" - ] - }, { "cell_type": "code", "execution_count": 112, @@ -24724,9 +24711,11 @@ }, "outputs": [], "source": [ + "conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")\n", + "\n", "collection = 'SAR2Cube_SInCohMap_S1_L0_147_ASC_DONYANA'\n", "# collection = 'SAR2Cube_SInCohMap_S1_L0_154_DSC_DONYANA'\n", - "# spatial_extent = {'west':bbox[0],'east':bbox[2],'south':bbox[1],'north':bbox[3]}\n", + "spatial_extent = {'west':bbox[0],'east':bbox[2],'south':bbox[1],'north':bbox[3]}\n", "temporal_extent = [\"2018-06-01T00:00:00.000Z\", \"2018-06-20T00:00:00.000Z\"]\n", "\n", "S1_slant_range = conn.load_collection(collection,spatial_extent=spatial_extent,temporal_extent=temporal_extent)" @@ -24991,8 +24980,10 @@ "metadata": {}, "outputs": [], "source": [ - "collection = 'SAR2Cube_L0_117_ASC_ST_2016_2020_IFG_LIA_DEM'\n", - "# spatial_extent = {'west':bbox[0],'east':bbox[2],'south':bbox[1],'north':bbox[3]}\n", + "conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")\n", + "\n", + "collection = 'SAR2Cube_SInCohMap_S1_L0_147_ASC_DONYANA'\n", + "spatial_extent = {'west':bbox[0],'east':bbox[2],'south':bbox[1],'north':bbox[3]}\n", "temporal_extent = [\"2018-01-01T00:00:00.000Z\", \"2019-01-01T00:00:00.000Z\"]\n", "\n", "S1_slant_range = conn.load_collection(collection,spatial_extent=spatial_extent,temporal_extent=temporal_extent)\n", @@ -25023,7 +25014,7 @@ "\n", "S1_INT_ML_MEAN = S1_INT_ML.reduce_dimension(reducer=mean, dimension='DATE')\n", "\n", - "args_geocoding = {'resolution':20,'crs':32632}\n", + "args_geocoding = {'resolution':20,'crs':32629}\n", "S1_INT_ML_GEOCODED = S1_INT_ML_MEAN.process(\"geocode\",args_geocoding, data=S1_INT_ML_MEAN)\n", "\n", "S1_INT_ML_GEOCODED_GTIFF = S1_INT_ML_GEOCODED.save_result(format='GTiff')" @@ -25062,7 +25053,6 @@ "metadata": {}, "outputs": [], "source": [ - "job_id = \"8bd3844b-2f2e-4aa6-96bc-2841e98315fc\"\n", "job = conn.job(job_id)" ] }, @@ -39081,6 +39071,69 @@ "# Coherence computation on-the-fly" ] }, + { + "cell_type": "markdown", + "id": "7d2cef35-a659-40a0-98d6-f7da59ae5886", + "metadata": {}, + "source": [ + "## Select the AOI\n", + "Use the rectangle selection tool to select the area of interest" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "910c431a-6e89-48bc-8f1a-5cfe50dad84c", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "21197eec1bd14ee98ab9fbcb37e6eae6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map(center=[37.2, -6.2], controls=(ZoomControl(options=['position', 'zoom_in_text', 'zoom_in_title', 'zoom_out…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "center = [37.2, -6.2]\n", + "zoom = 9\n", + "\n", + "eoMap = openeoMap(center,zoom)\n", + "eoMap.map" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "0d486a20-0276-460a-86e9-70e16e525e2f", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Coordinates selected from map: \n", + " west -6.107025 \n", + " east -6.05175 \n", + " south 37.121392 \n", + " north 37.144658\n" + ] + } + ], + "source": [ + "bbox = eoMap.getBbox()\n", + "print(\"Coordinates selected from map:\",'\\n west',bbox[0],'\\n east',bbox[2],'\\n south',bbox[1],'\\n north',bbox[3])" + ] + }, { "cell_type": "markdown", "id": "pharmaceutical-success", @@ -39091,14 +39144,24 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 35, "id": "accompanied-setting", "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Authenticated using refresh token.\n" + ] + } + ], "source": [ - "collection = 'SAR2Cube_L0_117_ASC_ST_2016_2020_IFG_LIA_DEM'\n", + "conn = openeo.connect(openeoHost).authenticate_oidc(client_id=\"openEO_PKCE\")\n", + "\n", + "collection = 'SAR2Cube_SInCohMap_S1_L0_147_ASC_DONYANA'\n", "spatial_extent = {'west':bbox[0],'east':bbox[2],'south':bbox[1],'north':bbox[3]}\n", "temporal_extent = [\"2018-06-05T00:00:00.000Z\", \"2018-07-05T00:00:00.000Z\"]\n", "\n", @@ -39115,7 +39178,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 36, "id": "casual-husband", "metadata": {}, "outputs": [], @@ -39126,7 +39189,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 37, "id": "indian-basket", "metadata": {}, "outputs": [], @@ -39137,7 +39200,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 38, "id": "phantom-combination", "metadata": {}, "outputs": [], @@ -39148,7 +39211,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 39, "id": "cooked-entity", "metadata": {}, "outputs": [], @@ -39184,7 +39247,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 40, "id": "similar-mathematics", "metadata": { "scrolled": true @@ -39196,7 +39259,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 41, "id": "tutorial-package", "metadata": { "scrolled": true @@ -39217,7 +39280,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 42, "id": "phantom-lesbian", "metadata": { "scrolled": true @@ -39229,37 +39292,114 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 44, "id": "quality-programming", "metadata": { "scrolled": true }, "outputs": [], "source": [ - "args_geocoding = {'resolution':20,'crs':32632}\n", - "S1_COH_GEOCODED = S1_COH.process(\"geocode\",args_geocoding, data=S1_COH)" + "args_geocoding = {'resolution':60,'crs':32629}\n", + "S1_COH_GEOCODED = S1_COH.process(\"geocode\",args_geocoding, data=S1_COH)\n", + "S1_COH_GEOCODED_NC = S1_COH_GEOCODED.save_result(format='netCDF')" ] }, { "cell_type": "code", - "execution_count": 132, - "id": "daily-hometown", - "metadata": { - "scrolled": true - }, + "execution_count": 45, + "id": "ef7ba6d5-fa19-4213-894a-c43c26e5f500", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 305 ms, sys: 68.7 ms, total: 373 ms\n", - "Wall time: 38.2 s\n" + "Batch job created with id: 30f23049-788e-4cc1-8ecc-6da69db4f9c5\n" ] } ], "source": [ - "%%time\n", - "S1_COH_GEOCODED.download(\"./data/S1_COH_6DAYS_GEOCODED.nc\",format='netCDF')" + "job = conn.create_job(S1_COH_GEOCODED_NC,title=\"Donyana_6D_coherence\")\n", + "job_id = job.job_id\n", + "if job_id:\n", + " print(\"Batch job created with id: \",job_id)\n", + " job.start_job()\n", + "else:\n", + " print(\"Error! Job ID is None\")" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "dfa2ec8a-5636-4ba8-843a-0303a18f7baf", + "metadata": {}, + "outputs": [], + "source": [ + "job = conn.job(job_id)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "4bca73dd-1b12-46c4-b409-cfa9619812ec", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "d3739f24-4b15-4f9b-9cb7-be0ffd82a3cf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[PosixPath('job_result/process.json'), PosixPath('job_result/result.nc')]" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result = job.get_results()\n", + "result.download_files(\"./data/\")" ] }, {