copy data between two elastic search indices with different mappings but same fields

Question

I have an index1 with the following mapping:

{
    "index1": {
        "mappings": {
            "properties": {
                "school_id": {
                    "type": "text",
                    "fields": {
                        "keyword": {
                            "type": "keyword"
                        }
                    }
                },
                "cases": {
                    "type": "nested",
                    "properties": {
                        "teacher_id": {
                            "type": "text",
                            "fields": {
                                "keyword": {
                                    "type": "keyword"
                                }
                            }
                        },
                        "reports": {
                            "type": "nested",
                            "properties": {
                                "name": {
                                    "type": "text"
                                },
                                "height": {
                                    "type": "text"
                                },
                                "family_name": {
                                    "type": "text"
                                },
                                "studentID": {
                                    "type": "keyword"
                                },
                                "_class": {
                                    "type": "keyword",
                                    "index": false,
                                    "doc_values": false
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

And I have another index2 with the folowing mapping:

{
    "index2": {
        "mappings": {
            "properties": {
                "teacher_id": {
                    "type": "text"
                },
                "school_id": {
                    "type": "text"
                },
                "name": {
                    "type": "text"
                },
                "height": {
                    "type": "text"
                },
                "family_name": {
                    "type": "text"
                },
                "studentID": {
                    "type": "keyword"
                },                
                "_class": {
                    "type": "keyword",
                    "index": false,
                    "doc_values": false
                }
            }
        }
    }
}

Now I want to copy the values of the fields from index2 to index1. For example value of teacher_id in index2 should just copy to teacher_id in index1 without altering their mapping structures. Please note that index1 has a nested mapping structure and index2 has a plain mapping. How can this be achieved?

I tried the following solution

POST /_reindex
{    
    "source": {
        "index": "index2"
    },
    "dest": {
        "index": "index1"
    },
    "script": {
        "source": """
            ctx._source.school_id = params._source.school_id;
            ctx._source.cases.teacher_id = params._source.teacher_id;
            ctx._source.cases.reports.studentID = params._source.studentID;
            ......
          """
    }
}

but I get the following error:

{
    "error": {
        "root_cause": [
            {
                "type": "script_exception",
                "reason": "runtime error",
                "script_stack": [
                    """
                    ctx._source.school_id = params._source.school_id;
                    """,
                    "                                         ^---- HERE"
                 ],
                "script": " ...",
                "lang": "painless",
                "position": {
                    "offset": 48,
                    "start": 7,
                    "end": 69
                }
           }
        ],
        "type": "script_exception",
        "reason": "runtime error",
        "script_stack": [
            """ctx._source.school_id = params._source.school_id;
            """,
            "                                         ^---- HERE"
        ],
        "script": " ...",
        "lang": "painless",
        "position": {
            "offset": 48,
            "start": 7,
            "end": 69
        },
        "caused_by": {
            "type": "null_pointer_exception",
            "reason": "cannot access method/field [school_id] from a null def reference"
        }
    },
    "status": 400
}

First of all I am not even sure if the solution is correct. May I know the correct solution?

Answer 1

I recommend to use ingest pipeline.

I created an ingest pipeline for the teached_id field in index2 it will index into index1 cases.teached_id

Here is an example for you only for teached_id, you can add more for each field.

PUT index1
{
  "mappings": {
    "properties": {
      "school_id": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        }
      },
      "cases": {
        "type": "nested",
        "properties": {
          "teacher_id": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "reports": {
            "type": "nested",
            "properties": {
              "name": {
                "type": "text"
              },
              "height": {
                "type": "text"
              },
              "family_name": {
                "type": "text"
              },
              "studentID": {
                "type": "keyword"
              },
              "_class": {
                "type": "keyword",
                "index": false,
                "doc_values": false
              }
            }
          }
        }
      }
    }
  }
}

---

PUT index2
{
  "mappings": {
    "properties": {
      "teacher_id": {
        "type": "text"
      },
      "school_id": {
        "type": "text"
      },
      "name": {
        "type": "text"
      },
      "height": {
        "type": "text"
      },
      "family_name": {
        "type": "text"
      },
      "studentID": {
        "type": "keyword"
      },
      "_class": {
        "type": "keyword",
        "index": false,
        "doc_values": false
      }
    }
  }
}

---

POST index2/_bulk
{"index": {}}
{"teacher_id": "t1", "school_id": "s1"}

---

PUT _ingest/pipeline/teacher_id_pipeline
{
  "processors": [
    {
      "set": {
        "field": "cases.teacher_id",
        "value": "{{teacher_id}}"
      }
    },
    {
      "remove": {
        "field": "teacher_id"
      }
    }
  ]
}

---

POST _reindex
{
  "source": {
    "index": "index2"
  },
  "dest": {
    "index": "index1",
    "pipeline": "teacher_id_pipeline"
  }
}

---

GET index2/_search
GET index1/_search