/
Elasticsearch Upgrade
Elasticsearch Upgrade
- Dustin Lo
Owned by Dustin Lo
Version: 1.7 → 7.1
Big Changes
Only 1 type allowed in each index:
_docNeed to manually enable all field text searches
Removal of
filteredsince ES 5.0Changes in the geo coordinates query
Note:
{"type": "geo_shape","tree": "quadtree","tree_levels": 26}makes uploading documents slow, specifically"tree_levels”: 26{ "query": { "bool": { "filter": { "geo_shape": { "location": { "shape": { "type": "polygon", "coordinates": [[<coordinates>]] }, "relation": "within" } } } } } }
Removal of
_all: { "enabled": true }type in indices so we cannot search for all fieldsworkaround is adding
copy_toin field mapping, especially in dynamic templating
Running Elasticsearch 7 on EC2 instance
In order to expose port 0.0.0.0 properly, we need to edit the config/elasticsearch.yml file
network.host: 0.0.0.0
cluster.name: grq_cluster
node.name: ESNODE_CYR
node.master: true
node.data: true
transport.host: localhost
transport.tcp.port: 9300
http.port: 9200
discovery.zen.minimum_master_nodes: 2
# allows UI to talk to elasticsearch (in production we would put the actual hostname of the uI)
http.cors.enabled : true
http.cors.allow-origin: "*"
Running Kibana on EC2 instance
Install Kibana in command line
curl -O https://artifacts.elastic.co/downloads/kibana/kibana-7.1.1-darwin-x86_64.tar.gz
tar -xzf kibana-7.1.1-darwin-x86_64.tar.gz
cd kibana-7.1.1-darwin-x86_64/ Edit the config/kibana.yml file to expose host 0.0.0.0
server.host: 0.0.0.0Index Mapping
Index: grq_v1.1_s1-iw_slc
What I did to update the mapping
create mapping called grq_v1.1_s1-iw_slc
copied example data into new ES index, using built in dynamic mapping to build initial mapping
mapping changes:
metadata.contextto{"type": "object", "enabled": false}properties.locationto{"type": "geo_shape","tree": "quadtree"}use type
keywordto be able to usemsearch:"reason": "Fielddata is disabled on text fields by default. ... Alternatively use a keyword field instead."
PUT grq_v1.1_s1-iw_slc/_mapping
{
"properties": {
"browse_urls": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"center": {
"type": "geo_shape",
"tree": "quadtree"
},
"city": {
"properties": {
"admin1_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"admin1_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"admin2_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"admin2_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"alternatename": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"asciiname": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"continent_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"continent_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"country_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"country_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dem": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"elevation": {
"type": "long"
},
"feature_class": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"feature_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"geonameid": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"latitude": {
"type": "float"
},
"location": {
"properties": {
"lat": {
"type": "float"
},
"lon": {
"type": "float"
}
}
},
"longitude": {
"type": "float"
},
"modification_date": {
"type": "date",
"format": "date_optional_time"
},
"name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"population": {
"type": "long"
},
"timezone": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent": {
"type": "keyword",
"ignore_above": 256
},
"dataset": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dataset_level": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dataset_type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"endtime": {
"type": "date",
"format": "date_optional_time"
},
"id": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"images": {
"properties": {
"img": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"small_img": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"tooltip": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
},
"ipath": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"label": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"location": {
"type": "geo_shape",
"tree": "quadtree"
},
"metadata": {
"properties": {
"archive_filename": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"bbox": {
"type": "float"
},
"context": {
"type": "object",
"enabled": false
},
"data_product_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"direction": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"downlinkStart": {
"type": "date",
"format": "date_optional_time"
},
"downlinkStop": {
"type": "date",
"format": "date_optional_time"
},
"download_url": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"l0ProcessingStart": {
"type": "date",
"format": "date_optional_time"
},
"l0ProcessingStop": {
"type": "date",
"format": "date_optional_time"
},
"orbitCycle": {
"type": "long"
},
"orbitNumber": {
"type": "long"
},
"orbitRepeat": {
"type": "long"
},
"platform": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"postProcessingStart": {
"type": "date",
"format": "date_optional_time"
},
"postProcessingStop": {
"type": "date",
"format": "date_optional_time"
},
"processingStart": {
"type": "date",
"format": "date_optional_time"
},
"processingStop": {
"type": "date",
"format": "date_optional_time"
},
"sensingStart": {
"type": "date",
"format": "date_optional_time"
},
"sensingStop": {
"type": "date",
"format": "date_optional_time"
},
"sensor": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"trackNumber": {
"type": "long"
},
"version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
},
"objectid": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"starttime": {
"type": "date",
"format": "date_optional_time"
},
"system_version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"temporal_span": {
"type": "long"
},
"urls": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
}
Index: grq_v3.0_area_of_interest
PUT grq_v3.0_area_of_interest/_mapping
{
"properties": {
"account": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"browse_urls": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"center": {
"properties": {
"coordinates": {
"type": "float"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"city": {
"properties": {
"admin1_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"admin1_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"admin2_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"admin2_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"alternatename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"asciiname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dem": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"elevation": {
"type": "long"
},
"feature_class": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"feature_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"geonameid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"latitude": {
"type": "float"
},
"location": {
"properties": {
"lat": {
"type": "float"
},
"lon": {
"type": "float"
}
}
},
"longitude": {
"type": "float"
},
"modification_date": {
"type": "date",
"format": "date_optional_time"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"population": {
"type": "long"
},
"timezone": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"continent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dataset": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dataset_level": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dataset_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"emails": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"endtime": {
"type": "date",
"format": "date_optional_time"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ipath": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"location": {
"type": "geo_shape",
"tree": "quadtree"
},
"metadata": {
"properties": {
"context": {
"type": "object",
"enabled": false
},
"eventtime": {
"type": "date",
"format": "date_optional_time"
},
"username": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"objectid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"prov": {
"properties": {
"wasGeneratedBy": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"starttime": {
"type": "date",
"format": "date_optional_time"
},
"system_version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"temporal_span": {
"type": "long"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"urls": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
Index: grq_v2.0.2_s1-gunw
PUT grq_v2.0.2_s1-gunw/_mapping
{
"properties": {
"browse_urls": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"center": {
"properties": {
"coordinates": {
"type": "float"
},
"type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
},
"city": {
"properties": {
"admin1_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"admin1_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"admin2_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"admin2_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"asciiname": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"continent_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"continent_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"country_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"country_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dem": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"elevation": {
"type": "long"
},
"feature_class": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"feature_code": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"geonameid": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"latitude": {
"type": "float"
},
"location": {
"properties": {
"lat": {
"type": "float"
},
"lon": {
"type": "float"
}
}
},
"longitude": {
"type": "float"
},
"modification_date": {
"type": "date"
},
"name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"population": {
"type": "long"
},
"timezone": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
},
"continent": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"creation_timestamp": {
"type": "date"
},
"dataset": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dataset_level": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dataset_type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"endtime": {
"type": "date"
},
"id": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"ipath": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"label": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"location": {
"type": "geo_shape",
"tree": "quadtree"
},
"metadata": {
"properties": {
"archive_filename": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"azimuth_looks": {
"type": "long"
},
"bbox": {
"type": "float"
},
"beam_mode": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"context": {
"type": "object",
"enabled": false
},
"dataset_type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"dem_type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"doppler": {
"type": "float"
},
"esd_threshold": {
"type": "float"
},
"filter_strength": {
"type": "float"
},
"frame_id": {
"type": "long"
},
"frame_name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"full_id_hash": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"image_corners": {
"properties": {
"maxLat": {
"type": "float"
},
"maxLon": {
"type": "float"
},
"minLat": {
"type": "float"
},
"minLon": {
"type": "float"
}
}
},
"input_file": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"ipf_version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"latitude_index_max": {
"type": "long"
},
"latitude_index_min": {
"type": "long"
},
"look_direction": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"ogr_bbox": {
"type": "float"
},
"orbit_direction": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"orbit_number": {
"type": "long"
},
"orbit_repeat": {
"type": "long"
},
"orbit_type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"parallel_baseline": {
"type": "float"
},
"perpendicular_baseline": {
"type": "float"
},
"platform": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"polarization": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"prf": {
"type": "float"
},
"product_type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"range_looks": {
"type": "long"
},
"refbbox": {
"type": "float"
},
"reference_date": {
"type": "date"
},
"reference_scenes": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"scene_count": {
"type": "long"
},
"secondary_date": {
"type": "date"
},
"secondary_scenes": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"sensing_start": {
"type": "date"
},
"sensing_stop": {
"type": "date"
},
"sensor": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"sha224sum": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"starting_range": {
"type": "float"
},
"system_version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"tags": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"temporal_span": {
"type": "long"
},
"tile_layers": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"tiles": {
"type": "boolean"
},
"track_number": {
"type": "long"
},
"union_geojson": {
"properties": {
"coordinates": {
"type": "float"
},
"type": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
}
}
},
"objectid": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"prov": {
"properties": {
"wasDerivedFrom": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"wasGeneratedBy": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
},
"starttime": {
"type": "date"
},
"system_version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"temporal_span": {
"type": "long"
},
"urls": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
},
"version": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"raw": {
"type": "text"
}
}
}
}
}Related content
Ingest missing acquisitions for an AOI
Ingest missing acquisitions for an AOI
More like this
IPF Scraper SciHub
IPF Scraper SciHub
More like this
AOI based Submitter for Acquisition Scraper
AOI based Submitter for Acquisition Scraper
More like this
SAR-Availability Developers Guide (BOS Sarcat)
SAR-Availability Developers Guide (BOS Sarcat)
More like this
Create AOI
Create AOI
More like this
S1-GCOV pipeline
S1-GCOV pipeline
More like this