{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "http://azureml/sdk-2-0/MLTable.json",
  "title": "MLTable specification",
  "description": "An MLTable defines a set of instructions in order to build a table of data.",
  "type": "object",
  "properties": {
    "$schema": {
      "type": "string"
    },
    "type": {
      "const": "mltable"
    },
    "metadata": {
      "description": "Infomation about the MLTable",
      "type": "object",
      "additionalProperties": true
    },
    "paths": {
      "type": "array",
      "items": {
        "anyOf": [
          {
            "type": "object",
            "properties": {
              "file": {
                "type": "string",
                "description": "This specifies path of file containing data."
              }
            },
            "additionalProperties": false,
            "required": [
              "file"
            ]
          },
          {
            "type": "object",
            "properties": {
              "folder": {
                "type": "string",
                "description": "This specifies path of folder containing data."
              }
            },
            "additionalProperties": false,
            "required": [
              "folder"
            ]
          },
          {
            "type": "object",
            "properties": {
              "pattern": {
                "type": "string",
                "description": "This specifies a search pattern to allow globbing of files and folders containing data."
              }
            },
            "additionalProperties": false,
            "required": [
              "pattern"
            ]
          }
        ]
      }
    },
    "traits": {
      "type": "object",
      "properties": {
        "index_columns": {
          "type": "array",
          "description": "Sets the table index using existing columns. This trait can be used by partition_by in the data plane to split data by the index.",
          "items": {
            "type": "string"
          }
        },
        "timestamp_column": {
          "type": "string",
          "description": "Defines the timestamp column of the table. This trait can be used in filter transforms, or in other data plane operations (SDK) such as drift detection."
        }
      }
    },
    "transformations": {
      "description": "Defined sequence of transformations that are applied to data loaded from defined paths.",
      "type": "array",
      "items": {
        "oneOf": [
          {
            "const": "read_delimited"
          },
          {
            "const": "read_parquet"
          },
          {
            "const": "read_json_lines"
          },
          {
            "const": "read_delta_lake"
          },
          {
            "const": "extract_columns_from_partition_format"
          },
          {
            "type": "object",
            "properties": {
              "read_delimited": {
                "description": "Reads delimited files from provided paths, splitting them into rows and columns based on separator.",
                "type": "object",
                "properties": {
                  "delimiter": {
                    "type": "string",
                    "minLength": 1,
                    "maxLength": 2,
                    "default": ","
                  },
                  "header": {
                    "type": "string",
                    "enum": [
                      "no_header",
                      "from_first_file",
                      "all_files_different_headers",
                      "all_files_same_headers"
                    ],
                    "default": "all_files_same_headers"
                  },
                  "support_multi_line": {
                    "type": "boolean",
                    "default": false
                  },
                  "empty_as_string": {
                    "type": "boolean",
                    "default": false
                  },
                  "encoding": {
                    "type": "string",
                    "enum": [
                      "utf8",
                      "iso88591",
                      "latin1",
                      "ascii",
                      "utf16",
                      "utf32",
                      "utf8bom",
                      "windows1252"
                    ],
                    "default": "utf8"
                  },
                  "include_path_column": {
                    "type": "boolean",
                    "default": false
                  },
                  "infer_column_types": {
                    "type": [
                      "object",
                      "boolean"
                    ],
                    "properties": {
                      "sample_size": {
                        "type": "integer",
                        "default": 100
                      },
                      "column_type_overrides": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "columns": {
                              "oneOf": [
                                {
                                  "type": "string",
                                  "description": "single column to be selected"
                                },
                                {
                                  "type": "array",
                                  "description": "list of columns to be selected",
                                  "items": {
                                    "type": "string"
                                  }
                                }
                              ]
                            },
                            "column_type": {
                              "oneOf": [
                                {
                                  "type": "string",
                                  "enum": [
                                    "boolean",
                                    "float",
                                    "string",
                                    "int",
                                    "stream_info"
                                  ]
                                },
                                {
                                  "type": "object",
                                  "properties": {
                                    "boolean": {
                                      "type": "object",
                                      "properties": {
                                        "true_values": {
                                          "type": "array",
                                          "items": {
                                            "type": "string"
                                          }
                                        },
                                        "false_values": {
                                          "type": "array",
                                          "items": {
                                            "type": "string"
                                          }
                                        },
                                        "mismatch_as": {
                                          "type": "string",
                                          "enum": [
                                            "false",
                                            "true",
                                            "error"
                                          ],
                                          "default": "error"
                                        }
                                      },
                                      "additionalProperties": false
                                    }
                                  },
                                  "additionalProperties": false
                                },
                                {
                                  "type": "object",
                                  "properties": {
                                    "datetime": {
                                      "type": "object",
                                      "properties": {
                                        "formats": {
                                          "type": "array",
                                          "items": {
                                            "type": "string"
                                          }
                                        },
                                        "date_constant": {
                                          "type": "string"
                                        }
                                      },
                                      "required": [
                                        "formats"
                                      ],
                                      "additionalProperties": false
                                    }
                                  },
                                  "additionalProperties": false
                                }
                              ]
                            }
                          },
                          "required": [
                            "columns",
                            "column_type"
                          ],
                          "additionalProperties": false
                        }
                      },
                      "additionalProperties": false
                    },
                    "default": true,
                    "additionalProperties": false
                  },
                  "path_column": {
                    "type": "string"
                  },
                  "partition_size": {
                    "type": "integer"
                  }
                },
                "additionalProperties": false
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "read_parquet": {
                "description": "Reads parquet files from the provided paths using the defined options.",
                "type": "object",
                "properties": {
                  "include_path_column": {
                    "type": "boolean",
                    "default": false
                  },
                  "path_column": {
                    "type": "string"
                  }
                },
                "additionalProperties": false
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "read_json_lines": {
                "description": "Reads JSON lines file from the provided paths using the defined options.",
                "type": "object",
                "properties": {
                  "invalid_lines": {
                    "type": "string",
                    "enum": [
                      "error",
                      "drop"
                    ],
                    "default": "error"
                  },
                  "encoding": {
                    "type": "string",
                    "enum": [
                      "utf8",
                      "iso88591",
                      "latin1",
                      "ascii",
                      "utf16",
                      "utf32",
                      "utf8bom",
                      "windows1252"
                    ],
                    "default": "utf8"
                  },
                  "include_path_column": {
                    "type": "boolean",
                    "default": false
                  },
                  "path_column": {
                    "type": "string"
                  },
                  "partition_size": {
                    "type": "integer"
                  }
                },
                "additionalProperties": false
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "read_delta_lake": {
                "description": "Reads parquet files from delta lake using the defined options.",
                "type": "object",
                "properties": {
                  "version_as_of": {
                    "type": "integer",
                    "description": "integer version to use to read in a specific version of parquet files."
                  },
                  "timestamp_as_of": {
                    "type": "string",
                    "description": "datetime string in RFC-3339/ISO-8601 format to use to read in matching parquet files from a specific point in time."
                  },
                  "include_path_column": {
                    "type": "boolean",
                    "default": false
                  }
                },
                "additionalProperties": false
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "extract_columns_from_partition_format": {
                "type": "object",
                "description": "The partition information of each path will be extracted into columns based on the specified format",
                "properties": {
                  "partition_format": {
                    "type": "string",
                    "description": "Partition format to use to extract data into columns"
                  }
                }
              }
            },
            "additionalProperties": false
          },
		  {
            "type": "object",
            "properties": {
              "filter": {
                "description": "filter rows based on the expression",
                "type": "string"
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "take": {
                "description": "take the first N rows from dataset",
                "type": "integer"
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "take_random_sample": {
                "description": "take a random sample of records in the dataset approximately by the probability specified",
                "type": "object",
                "properties": {
                  "probability": {
                    "type": "number",
                    "minimum": 0.0,
                    "maximum": 1.0
                  },
                  "seed": {
                    "type": "integer"
                  }
                }
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "skip": {
                "description": "Skip the first N records from top of the dataset",
                "type": "integer"
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "drop_columns": {
                "description": "Drop the specified columns from the dataset",
                "oneOf": [
                  {
                    "type": "string",
                    "description": "single column to be selected"
                  },
                  {
                    "type": "array",
                    "description": "list of columns to be selected",
                    "items": {
                      "type": "string"
                    }
                  },
                  {
                    "type": "object",
                    "description": "pattern and properties defining columns to be selected",
                    "properties": {
                      "pattern": {
                        "type": "string"
                      },
                      "ignore_case": {
                        "type": "boolean",
                        "default": false
                      },
                      "invert": {
                        "type": "boolean",
                        "default": false
                      }
                    }
                  }
                ]
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "keep_columns": {
                "description": "Keep only the specified columns from the dataset",
                "oneOf": [
                  {
                    "type": "string",
                    "description": "single column to be selected"
                  },
                  {
                    "type": "array",
                    "description": "list of columns to be selected",
                    "items": {
                      "type": "string"
                    }
                  },
                  {
                    "type": "object",
                    "description": "pattern and properties defining columns to be selected",
                    "properties": {
                      "pattern": {
                        "type": "string"
                      },
                      "ignore_case": {
                        "type": "boolean",
                        "default": false
                      },
                      "invert": {
                        "type": "boolean",
                        "default": false
                      }
                    }
                  }
                ]
              }
            },
            "additionalProperties": false
          },
          {
            "type": "object",
            "properties": {
              "convert_column_types": {
                "description": "Convert the columns specified into the column_type desired",
                "type": "array",
                "items": {
                  "oneOf": [
                    {
                      "type": "object",
                      "properties": {
                        "columns": {
                          "oneOf": [
                            {
                              "type": "string",
                              "description": "single column to be selected"
                            },
                            {
                              "type": "array",
                              "description": "list of columns to be selected",
                              "items": {
                                "type": "string"
                              }
                            },
                            {
                              "type": "object",
                              "description": "pattern and properties defining columns to be selected",
                              "properties": {
                                "pattern": {
                                  "type": "string"
                                },
                                "ignore_case": {
                                  "type": "boolean",
                                  "default": false
                                },
                                "invert": {
                                  "type": "boolean",
                                  "default": false
                                }
                              }
                            }
                          ]
                        },
                        "column_type": {
                          "oneOf": [
                            {
                              "type": "string",
                              "enum": [
                                "boolean",
                                "float",
                                "string",
                                "int",
                                "stream_info"
                              ]
                            },
                            {
                              "type": "object",
                              "properties": {
                                "datetime": {
                                  "type": "object",
                                  "properties": {
                                    "formats": {
                                      "type": "array",
                                      "items": {
                                        "type": "string"
                                      }
                                    },
                                    "date_constant": {
                                      "type": "string"
                                    }
                                  },
                                  "required": [
                                    "formats"
                                  ],
                                  "additionalProperties": false
                                }
                              },
                              "additionalProperties": false
                            },
                            {
                              "type": "object",
                              "properties": {
                                "float": {
                                  "type": "object",
                                  "properties": {
                                    "decimal_mark": {
                                      "type": "string",
                                      "enum": [
                                        ".",
                                        ","
                                      ],
                                      "default": "."
                                    }
                                  },
                                  "additionalProperties": false
                                }
                              },
                              "additionalProperties": false
                            },
                            {
                              "type": "object",
                              "properties": {
                                "boolean": {
                                  "type": "object",
                                  "properties": {
                                    "true_values": {
                                      "type": "array",
                                      "items": {
                                        "type": "string"
                                      }
                                    },
                                    "false_values": {
                                      "type": "array",
                                      "items": {
                                        "type": "string"
                                      }
                                    },
                                    "mismatch_as": {
                                      "type": "string",
                                      "enum": [
                                        "false",
                                        "true",
                                        "error"
                                      ],
                                      "default": "error"
                                    }
                                  },
                                  "additionalProperties": false
                                }
                              },
                              "additionalProperties": false
                            }
                          ]
                        }
                      },
                      "additionalProperties": false
                    }
                  ]
                }
              }
            },
            "additionalProperties": false
          }
        ]
      }
    }
  },
  "additionalProperties": false,
  "oneOf": [
    {
      "required": [
        "paths"
      ]
    }
  ],
  "definitions": {
    "convert_column_types": {
      "description": "Convert the columns specified into the column_type desired",
      "type": "array",
      "items": {
        "oneOf": [
          {
            "type": "object",
            "properties": {
              "columns": {
                "oneOf": [
                  {
                    "type": "string",
                    "description": "single column to be selected"
                  },
                  {
                    "type": "array",
                    "description": "list of columns to be selected",
                    "items": {
                      "type": "string"
                    }
                  },
                  {
                    "type": "object",
                    "description": "pattern and properties defining columns to be selected",
                    "properties": {
                      "pattern": {
                        "type": "string"
                      },
                      "ignore_case": {
                        "type": "boolean",
                        "default": false
                      },
                      "invert": {
                        "type": "boolean",
                        "default": false
                      }
                    }
                  }
                ]
              },
              "column_type": {
                "oneOf": [
                  {
                    "type": "string",
                    "enum": [
                      "boolean",
                      "float",
                      "string",
                      "int",
                      "stream_info"
                    ]
                  },
                  {
                    "type": "object",
                    "properties": {
                      "datetime": {
                        "type": "object",
                        "properties": {
                          "formats": {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          "date_constant": {
                            "type": "string"
                          }
                        },
                        "required": [
                          "formats"
                        ],
                        "additionalProperties": false
                      }
                    },
                    "additionalProperties": false
                  },
                  {
                    "type": "object",
                    "properties": {
                      "float": {
                        "type": "object",
                        "properties": {
                          "decimal_mark": {
                            "type": "string",
                            "enum": [
                              ".",
                              ","
                            ],
                            "default": "."
                          }
                        },
                        "additionalProperties": false
                      }
                    },
                    "additionalProperties": false
                  },
                  {
                    "type": "object",
                    "properties": {
                      "boolean": {
                        "type": "object",
                        "properties": {
                          "true_values": {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          "false_values": {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          "mismatch_as": {
                            "type": "string",
                            "enum": [
                              "false",
                              "true",
                              "error"
                            ],
                            "default": "error"
                          }
                        },
                        "additionalProperties": false
                      }
                    },
                    "additionalProperties": false
                  }
                ]
              }
            },
            "additionalProperties": false
          }
        ]
      }
    },
    "column_selector": {
      "oneOf": [
        {
          "type": "string",
          "description": "single column to be selected"
        },
        {
          "type": "array",
          "description": "list of columns to be selected",
          "items": {
            "type": "string"
          }
        },
        {
          "type": "object",
          "description": "pattern and properties defining columns to be selected",
          "properties": {
            "pattern": {
              "type": "string"
            },
            "ignore_case": {
              "type": "boolean",
              "default": false
            },
            "invert": {
              "type": "boolean",
              "default": false
            }
          }
        }
      ]
    }
  }
}