compute_pipes

type AggregateTransformationPipe struct {
	// contains filtered or unexported fields
}

type AnalyzeSpec struct {
	SchemaProvider                  string               `json:"schema_provider,omitempty"`
	ScrubChars                      string               `json:"scrub_chars,omitempty"`
	DistinctValuesWhenLessThanCount int                  `json:"distinct_values_when_less_than_count,omitzero"`
	PadShortRowsWithNulls           bool                 `json:"pad_short_rows_with_nulls,omitzero"`
	ColumnNameToken                 *ColumnNameTokenNode `json:"column_name_token,omitempty"`
	EntityHints                     []*EntityHint        `json:"entity_hints,omitempty"`
	RegexTokens                     []RegexNode          `json:"regex_tokens,omitempty"`
	LookupTokens                    []LookupTokenNode    `json:"lookup_tokens,omitempty"`
	KeywordTokens                   []KeywordTokenNode   `json:"keyword_tokens,omitempty"`
	FunctionTokens                  []FunctionTokenNode  `json:"function_tokens,omitempty"`
}

type AnalyzeState struct {
	ColumnName     string
	ColumnPos      int
	DistinctValues map[string]*DistinctCount
	NullCount      int
	LenWelford     *WelfordAlgo
	CharToScrub    map[rune]bool
	RegexMatch     map[string]*RegexCount
	LookupState    []*LookupTokensState
	KeywordMatch   map[string]*KeywordCount
	ParseDate      *ParseDateMatchFunction
	ParseDouble    *ParseDoubleMatchFunction
	ParseText      *ParseTextMatchFunction
	TotalRowCount  int
	BlankMarkers   *BlankFieldMarkers
	Spec           *TransformationSpec
}

type AnalyzeTransformationPipe struct {
	// contains filtered or unexported fields
}

type AnonymizationAction struct {
	// contains filtered or unexported fields
}

type AnonymizeSpec struct {
	Mode                        string               `json:"mode,omitempty"`
	LookupName                  string               `json:"lookup_name,omitempty"`
	AnonymizeType               string               `json:"anonymize_type,omitempty"`
	KeyPrefix                   string               `json:"key_prefix,omitempty"`
	DeidFunctions               map[string]string    `json:"deid_functions,omitempty"`
	DeidLookups                 map[string]string    `json:"deid_lookups,omitempty"`
	InputDateLayout             string               `json:"input_date_layout,omitempty"`
	DateFormatsColumn           string               `json:"date_formats_column,omitempty"`
	OutputDateLayout            string               `json:"output_date_layout,omitempty"`
	KeyDateLayout               string               `json:"key_date_layout,omitempty"`
	DefaultInvalidDate          string               `json:"default_invalid_date,omitempty"`
	SchemaProvider              string               `json:"schema_provider,omitempty"`
	AdjustFieldWidthOnFW        bool                 `json:"adjust_field_width_on_fixed_width_file,omitzero"`
	OmitPrefixOnFW              bool                 `json:"omit_prefix_on_fixed_width_file,omitzero"`
	AnonymizedColumnsOutputFile *ColumnFileSpec      `json:"anonymized_columns_output_file,omitzero"`
	KeysOutputChannel           *OutputChannelConfig `json:"keys_output_channel"`
}

type AnonymizeTransformationPipe struct {
	// contains filtered or unexported fields
}

type ArrayBuilder interface {
	Reserve(n int)
	Append(v any)
	AppendEmptyValue()
	AppendNull()
	NewArray() arrow.Array
	Release()
}

type ArrayRecord struct {
	Record arrow.Record
	// contains filtered or unexported fields
}

type BadRowsChannel struct {
	OutputCh chan []byte
	// contains filtered or unexported fields
}

type BadRowsSpec struct {
	BadRowsStepId string `json:"bad_rows_step_id,omitempty"`
}

type BinaryBuilder struct {
	// contains filtered or unexported fields
}

type BlankFieldMarkers struct {
	CaseSensitive bool
	Markers       []string
}

type BlankFieldMarkersSpec struct {
	CaseSensitive bool     `json:"case_sensitive,omitzero"`
	Markers       []string `json:"markers,omitempty"`
}

type BooleanBuilder struct {
	// contains filtered or unexported fields
}

type BuilderContext struct {
	// contains filtered or unexported fields
}

type CaseExpression struct {
	When ExpressionNode    `json:"when"`
	Then []*ExpressionNode `json:"then"`
}

type CastToRdfFnc = func(v any) (any, error)

type CastToRdfTxtFnc = func(v string) (any, error)

type Channel struct {
	Name          string
	Channel       chan []any
	Columns       *map[string]int
	DomainKeySpec *DomainKeysSpec
	Config        *ChannelSpec
}

type ChannelRegistry struct {
	InputRowChannel      *InputChannel
	ComputeChannels      map[string]*Channel
	OutputTableChannels  []string
	ClosedChannels       map[string]bool
	ClosedChMutex        sync.Mutex
	DistributionChannels map[string]*[]string
}

type ChannelResults struct {
	LoadFromS3FilesResultCh chan LoadFromS3FilesResult
	Copy2DbResultCh         chan chan ComputePipesResult
	WritePartitionsResultCh chan chan ComputePipesResult
	S3PutObjectResultCh     chan ComputePipesResult
	JetrulesWorkerResultCh  chan chan JetrulesWorkerResult
	ClusteringResultCh      chan chan ClusteringResult
}

type ChannelSpec struct {
	Name                 string          `json:"name"`
	Columns              []string        `json:"columns"`
	ClassName            string          `json:"class_name,omitempty"`
	DirectPropertiesOnly bool            `json:"direct_properties_only,omitzero"`
	HasDynamicColumns    bool            `json:"has_dynamic_columns,omitzero"`
	SameColumnsAsInput   bool            `json:"same_columns_as_input,omitzero"`
	DomainKeys           map[string]any  `json:"domain_keys,omitempty"`
	DomainKeysInfo       *DomainKeysSpec `json:"domain_keys_spec,omitzero"`
	// contains filtered or unexported fields
}

type ChannelState struct {
	// contains filtered or unexported fields
}

type ClusterCorrelation struct {
	// contains filtered or unexported fields
}

type ClusterInfo struct {
	// contains filtered or unexported fields
}

type ClusterShardingInfo struct {
	TotalFileSize     int64 `json:"total_file_size"`
	MaxNbrPartitions  int   `json:"max_nbr_partitions"`
	NbrPartitions     int   `json:"nbr_partitions"`
	MultiStepSharding int   `json:"multi_step_sharding"`
}

type ClusterShardingSpec struct {
	AppliesToFormat             string  `json:"applies_to_format,omitempty"`
	WhenTotalSizeGe             int     `json:"when_total_size_ge_mb,omitzero"`
	MaxNbrPartitions            int     `json:"max_nbr_partitions,omitzero"`
	MultiStepShardingThresholds int     `json:"multi_step_sharding_thresholds,omitzero"`
	ShardSizeMb                 float64 `json:"shard_size_mb,omitzero"`
	ShardMaxSizeMb              float64 `json:"shard_max_size_mb,omitzero"`
	ShardSizeBy                 float64 `json:"shard_size_by,omitzero"`     // for testing only
	ShardMaxSizeBy              float64 `json:"shard_max_size_by,omitzero"` // for testing only
	S3WorkerPoolSize            int     `json:"s3_worker_pool_size,omitzero"`
	MaxConcurrency              int     `json:"max_concurrency,omitzero"`
}

type ClusterSpec struct {
	MaxNbrPartitions            int                   `json:"max_nbr_partitons,omitzero"`
	MultiStepShardingThresholds int                   `json:"multi_step_sharding_thresholds,omitzero"`
	DefaultShardSizeMb          float64               `json:"default_shard_size_mb,omitzero"`
	DefaultShardMaxSizeMb       float64               `json:"default_shard_max_size_mb,omitzero"`
	DefaultShardSizeBy          float64               `json:"default_shard_size_by,omitzero"`     // for testing only
	DefaultShardMaxSizeBy       float64               `json:"default_shard_max_size_by,omitzero"` // for testing only
	ShardOffset                 int                   `json:"shard_offset,omitzero"`
	DefaultMaxConcurrency       int                   `json:"default_max_concurrency,omitzero"`
	S3WorkerPoolSize            int                   `json:"s3_worker_pool_size,omitzero"`
	ClusterShardingTiers        []ClusterShardingSpec `json:"cluster_sharding_tiers,omitempty"`
	IsDebugMode                 bool                  `json:"is_debug_mode,omitzero"`
	KillSwitchMin               int                   `json:"kill_switch_min,omitzero"`
	ShardingInfo                *ClusterShardingInfo  `json:"sharding_info,omitzero"`
}

type ClusteringDistributor struct {
	// contains filtered or unexported fields
}

type ClusteringPoolManager struct {
	WorkersTaskCh chan []any

	WaitForDone *sync.WaitGroup
	// contains filtered or unexported fields
}

type ClusteringResult struct {
	Err error
}

type ClusteringSpec struct {
	MaxInputCount                int                     `json:"max_input_count,omitzero"`
	MinColumn1NonNilCount        int                     `json:"min_column1_non_null_count,omitzero"`
	MinColumn2NonNilCount        int                     `json:"min_column2_non_null_count,omitzero"`
	TargetColumnsLookup          TargetColumnsLookupSpec `json:"target_columns_lookup"`
	ClusterDataSubclassification []string                `json:"cluster_data_subclassification,omitempty"`
	SoloDataSubclassification    []string                `json:"solo_data_subclassification,omitempty"`
	TransitiveDataClassification []string                `json:"transitive_data_classification,omitempty"`
	IsDebug                      bool                    `json:"is_debug,omitzero"`
	CorrelationOutputChannel     *OutputChannelConfig    `json:"correlation_output_channel,omitzero"`
}

type ClusteringTransformationPipe struct {
	// contains filtered or unexported fields
}

type ClusteringWorker struct {
	// contains filtered or unexported fields
}

type ColumnCorrelation struct {
	// contains filtered or unexported fields
}

type ColumnFileSpec struct {
	// OutputLocation: custom file key.
	// Bucket is bucket or empty for jetstore one.
	// Delimiter: rune delimiter to use for output file.
	Bucket         string `json:"bucket,omitempty"`
	OutputLocation string `json:"output_location,omitempty"`
	SchemaProvider string `json:"schema_provider,omitempty"`
	Delimiter      rune   `json:"delimiter,omitzero"`
}

type ColumnNameLookupNode struct {
	Name                string   `json:"name"`
	ColumnNames         []string `json:"column_names,omitempty"`
	ColumnNameFragments []string `json:"column_name_fragments,omitempty"`
}

type ColumnNameTokenNode struct {
	Name   string                  `json:"name"`
	Lookup []*ColumnNameLookupNode `json:"lookup,omitempty"`
}

type CompiledPartFileComponent struct {
	ColumnName string
	Regex      *regexp.Regexp
}

type ComputePipesArgs struct {
	ComputePipesNodeArgs
	ComputePipesCommonArgs
}

type ComputePipesCommonArgs struct {
	CpipesMode            string                     `json:"cpipes_mode,omitempty"`
	Client                string                     `json:"client,omitempty"`
	Org                   string                     `json:"org,omitempty"`
	ObjectType            string                     `json:"object_type,omitempty"`
	FileKey               string                     `json:"file_key,omitempty"`
	SessionId             string                     `json:"session_id,omitempty"`
	MainInputStepId       string                     `json:"read_step_id,omitempty"`
	MergeFiles            bool                       `json:"merge_files"`
	InputSessionId        string                     `json:"input_session_id,omitempty"`
	SourcePeriodKey       int                        `json:"source_period_key"`
	ProcessName           string                     `json:"process_name,omitempty"`
	SourcesConfig         SourcesConfigSpec          `json:"sources_config"`
	DomainKeysSpecByClass map[string]*DomainKeysSpec `json:"domain_keys_by_class,omitempty"`
	PipelineConfigKey     int                        `json:"pipeline_config_key"`
	UserEmail             string                     `json:"user_email,omitempty"`
}

type ComputePipesConfig struct {
	CommonRuntimeArgs      *ComputePipesCommonArgs `json:"common_runtime_args,omitzero"`
	MetricsConfig          *MetricsSpec            `json:"metrics_config,omitzero"`
	ClusterConfig          *ClusterSpec            `json:"cluster_config,omitzero"`
	OutputTables           []*TableSpec            `json:"output_tables,omitempty"`
	OutputFiles            []OutputFileSpec        `json:"output_files,omitempty"`
	LookupTables           []*LookupSpec           `json:"lookup_tables,omitempty"`
	Channels               []ChannelSpec           `json:"channels,omitempty"`
	Context                []ContextSpec           `json:"context,omitempty"`
	SchemaProviders        []*SchemaProviderSpec   `json:"schema_providers,omitempty"`
	PipesConfig            []PipeSpec              `json:"pipes_config,omitempty"`
	ReducingPipesConfig    [][]PipeSpec            `json:"reducing_pipes_config,omitempty"`
	ConditionalPipesConfig []ConditionalPipeSpec   `json:"conditional_pipes_config,omitempty"`
}

type ComputePipesContext struct {
	ComputePipesArgs
	CpConfig              *ComputePipesConfig
	FileKeyComponents     map[string]any
	PartFileKeyComponents []CompiledPartFileComponent
	AddionalInputHeaders  []string
	EnvSettings           map[string]any
	SamplingCount         int
	JetStoreTempFolder    string
	InputFileKeys         [][]*FileKeyInfo
	JetRules              JetRulesProxy
	ChResults             *ChannelResults
	KillSwitch            chan struct{}
	Done                  chan struct{}
	ErrCh                 chan error
	FileNamesCh           []chan FileName
	DownloadS3ResultCh    chan DownloadS3Result // avoid to modify ChannelResult for now...
	S3DeviceMgr           *S3DeviceManager
	SchemaManager         *SchemaManager
}

type ComputePipesNodeArgs struct {
	NodeId             int    `json:"id"`
	JetsPartitionLabel string `json:"jp,omitempty"`
	PipelineExecKey    int    `json:"pe"`
}

type ComputePipesResult struct {
	// Table name can be jets_partition name
	// PartCount is nbr of file part in jets_partition
	TableName    string
	CopyRowCount int64
	PartsCount   int64
	Err          error
}

type ComputePipesRun struct {
	CpipesCommands       any                   `json:"cpipesCommands"`
	CpipesCommandsS3Key  string                `json:"cpipesCommandsS3Key,omitempty"`
	CpipesMaxConcurrency int                   `json:"cpipesMaxConcurrency"`
	StartReducing        StartComputePipesArgs `json:"startReducing"`
	IsLastReducing       bool                  `json:"isLastReducing"`
	NoMoreTask           bool                  `json:"noMoreTask"`
	UseECSReducingTask   bool                  `json:"useECSReducingTask"`
	ReportsCommand       []string              `json:"reportsCommand"`
	SuccessUpdate        map[string]any        `json:"successUpdate"`
	ErrorUpdate          map[string]any        `json:"errorUpdate"`
}

type ConditionalEnvVarEvaluator struct {
	// contains filtered or unexported fields
}

type ConditionalEnvVariable struct {
	CaseExpr []CaseExpression  `json:"case_expr,omitempty"` // case operator
	ElseExpr []*ExpressionNode `json:"else_expr,omitempty"` // case operator
}

type ConditionalPipeSpec struct {
	StepName        string                   `json:"step_name,omitempty"`
	UseEcsTasks     bool                     `json:"use_ecs_tasks,omitzero"`
	UseEcsTasksWhen *ExpressionNode          `json:"use_ecs_tasks_when,omitzero"`
	PipesConfig     []PipeSpec               `json:"pipes_config"`
	When            *ExpressionNode          `json:"when,omitzero"`
	AddlEnv         []ConditionalEnvVariable `json:"addl_env,omitempty"`
}

type ConditionalTransformationSpec struct {
	When ExpressionNode     `json:"when"`
	Then TransformationSpec `json:"then"`
}

type ContextSpec struct {
	// Type range: file_key_component, partfile_key_component
	Type string `json:"type,omitempty"`
	Key  string `json:"key,omitempty"`
	Expr string `json:"expr,omitempty"`
}

type CpipesStartup struct {
	CpConfig                      ComputePipesConfig         `json:"compute_pipes_config"`
	ProcessName                   string                     `json:"process_name,omitempty"`
	InputColumns                  []string                   `json:"input_columns,omitempty"`
	InputColumnsOriginal          []string                   `json:"input_columns_original,omitempty"`
	MainInputSchemaProviderConfig *SchemaProviderSpec        `json:"main_input_schema_provider_config,omitzero"`
	MainInputDomainKeysSpec       *DomainKeysSpec            `json:"main_input_domain_keys_spec,omitzero"`
	MainInputDomainClass          string                     `json:"main_input_domain_class,omitempty"`
	DomainKeysSpecByClass         map[string]*DomainKeysSpec `json:"domain_keys_spec_by_class,omitzero"`
	EnvSettings                   map[string]any             `json:"env_settings,omitzero"`
	PipelineConfigKey             int                        `json:"pipeline_config_key,omitempty"`
	InputSessionId                string                     `json:"input_session_id,omitempty"`
	SourcePeriodKey               int                        `json:"source_period_key,omitempty"`
	OperatorEmail                 string                     `json:"operator_email,omitempty"`
}

type CsvSourceS3 struct {
	// contains filtered or unexported fields
}

type CsvSourceSpec struct {
	// This is used for lookup tables and loading metadata in jetrules.
	// This is a single file source, the first file found is taken.
	// Type range: cpipes, csv_file (future)
	// Default values are taken from current pipeline
	// Format: csv, headerless_csv
	// Compression: none, snappy
	// MakeEmptyWhenNoFile: Do not make an error when no files
	// are found, make empty source. Default: generate an error when no files
	// are found in s3.
	Type                string `json:"type"`
	Format              string `json:"format,omitempty"`
	Compression         string `json:"compression,omitempty"`
	Delimiter           rune   `json:"delimiter,omitzero"`       // default ','
	ProcessName         string `json:"process_name,omitempty"`   // for cpipes
	ReadStepId          string `json:"read_step_id,omitempty"`   // for cpipes
	JetsPartitionLabel  string `json:"jets_partition,omitempty"` // for cpipes
	SessionId           string `json:"session_id,omitempty"`     // for cpipes
	ClassName           string `json:"class_name,omitempty"`     // used by jetrules_config
	MakeEmptyWhenNoFile bool   `json:"make_empty_source_when_no_files_found,omitzero"`
}

type DataSchemaSpec struct {
	Columns string `json:"column"`
	RdfType string `json:"rdf_type,omitempty"`
}

type DateBuilder struct {
	// contains filtered or unexported fields
}

type Decimal128Builder struct {
	Precision int32
	Scale     int32
	// contains filtered or unexported fields
}

type Decimal256Builder struct {
	Precision int32
	Scale     int32
	// contains filtered or unexported fields
}

type DefaultPF struct {
	// contains filtered or unexported fields
}

type DefaultSchemaProvider struct {
	// contains filtered or unexported fields
}

type DistinctCount struct {
	Value string
	Count int
}

type DistinctSpec struct {
	DistinctOn []string `json:"distinct_on,omitempty"`
}

type DistinctTransformationPipe struct {
	// contains filtered or unexported fields
}

type DomainKeyInfo struct {
	KeyExpr    []string `json:"key_expr,omitempty"`
	ObjectType string   `json:"object_type,omitempty"`
}

type DomainKeysSpec struct {
	HashingOverride string                    `json:"hashing_override,omitempty"`
	DomainKeys      map[string]*DomainKeyInfo `json:"domain_keys_info,omitempty"`
}

type DownloadS3Result struct {
	InputFilesCount int
	TotalFilesSize  int64
	Err             error
}

type EntityHint struct {
	Entity             string   `json:"entity"`
	NameFragments      []string `json:"column_name_fragments,omitempty"`
	ExclusionFragments []string `json:"exclusion_fragments,omitempty"`
}

type ExprBuilderContext map[string]any

type ExpressionNode struct {
	// Type is for leaf nodes: select, value
	// Name is for CaseExpression.Then and TransformationColumnSpec.ElseExpr
	// to indicate which column to set the calculated value
	Name      string          `json:"name,omitempty"` // TransformationColumnSpec case operator
	Type      string          `json:"type,omitempty"`
	Expr      string          `json:"expr,omitempty"`
	ExprList  []string        `json:"expr_list,omitempty"`
	AsRdfType string          `json:"as_rdf_type,omitempty"`
	Arg       *ExpressionNode `json:"arg,omitzero"`
	Lhs       *ExpressionNode `json:"lhs,omitzero"`
	Op        string          `json:"op,omitempty"`
	Rhs       *ExpressionNode `json:"rhs,omitzero"`
}

type FetchFileInfoResult struct {
	Headers      []string
	SepFlag      jcsv.Chartype
	Encoding     string
	EolByte      byte
	MultiColumns bool
}

type FieldInfo struct {
	Name             string `json:"name"`
	Type             string `json:"type"`
	Nullable         bool   `json:"nullable,omitzero"`
	DecimalPrecision int32  `json:"precision,omitzero"`
	DecimalScale     int32  `json:"scale,omitzero"`
}

type FileConfig struct {
	BadRowsConfig              *BadRowsSpec           `json:"bad_rows_config,omitzero"`
	BlankFieldMarkers          *BlankFieldMarkersSpec `json:"blank_field_markers,omitzero"`
	Bucket                     string                 `json:"bucket,omitempty"`
	Compression                string                 `json:"compression,omitempty"`
	Delimiter                  rune                   `json:"delimiter,omitzero"`
	DetectCrAsEol              bool                   `json:"detect_cr_as_eol,omitzero"`
	DetectEncoding             bool                   `json:"detect_encoding,omitzero"`
	DiscardFileHeaders         bool                   `json:"discard_file_headers,omitzero"`
	DomainClass                string                 `json:"domain_class,omitempty"`
	DomainKeys                 map[string]any         `json:"domain_keys,omitempty"`
	DropExcedentHeaders        bool                   `json:"drop_excedent_headers,omitzero"`
	Encoding                   string                 `json:"encoding,omitempty"`
	EnforceRowMaxLength        bool                   `json:"enforce_row_max_length,omitzero"`
	EnforceRowMinLength        bool                   `json:"enforce_row_min_length,omitzero"`
	EolByte                    byte                   `json:"eol_byte,omitzero"`
	FileKey                    string                 `json:"file_key,omitempty"`
	LookbackPeriods            string                 `json:"lookback_periods,omitzero"`
	FileName                   string                 `json:"file_name,omitempty"` // Type output
	FixedWidthColumnsCsv       string                 `json:"fixed_width_columns_csv,omitempty"`
	Format                     string                 `json:"format,omitempty"`
	InputFormatDataJson        string                 `json:"input_format_data_json,omitempty"`
	IsPartFiles                bool                   `json:"is_part_files,omitzero"`
	KeyPrefix                  string                 `json:"key_prefix,omitempty"`
	MainInputRowCount          int64                  `json:"main_input_row_count,omitzero"`
	MultiColumnsInput          bool                   `json:"multi_columns_input,omitzero"`
	NbrRowsInRecord            int64                  `json:"nbr_rows_in_record,omitzero"` // Format: parquet
	NoQuotes                   bool                   `json:"no_quotes,omitzero"`
	ParquetSchema              *ParquetSchemaInfo     `json:"parquet_schema,omitzero"`
	PutHeadersOnFirstPartition bool                   `json:"put_headers_on_first_partition,omitzero"`
	QuoteAllRecords            bool                   `json:"quote_all_records,omitzero"`
	ReadBatchSize              int64                  `json:"read_batch_size,omitzero"` // Format: parquet
	ReadDateLayout             string                 `json:"read_date_layout,omitempty"`
	ReorderColumnsOnRead       []int                  `json:"reorder_columns_on_read,omitempty"`
	TrimColumns                bool                   `json:"trim_columns,omitzero"`
	UseLazyQuotes              bool                   `json:"use_lazy_quotes,omitzero"`
	UseLazyQuotesSpecial       bool                   `json:"use_lazy_quotes_special,omitzero"`
	VariableFieldsPerRecord    bool                   `json:"variable_fields_per_record,omitzero"`
	WriteDateLayout            string                 `json:"write_date_layout,omitempty"`
	OutputEncoding             string                 `json:"output_encoding,omitempty"`
	OutputEncodingSameAsInput  bool                   `json:"output_encoding_same_as_input,omitempty"`
}

type FileKeyInfo struct {
	// contains filtered or unexported fields
}

type FileName struct {
	LocalFileName string
	InFileKeyInfo FileKeyInfo
}

type FilterColumnSpec struct {
	LookupName     string   `json:"lookup_name,omitempty"`
	ColumnName     string   `json:"column_name,omitempty"`
	LookupColumn   string   `json:"lookup_column,omitempty"`
	RetainOnValues []string `json:"retain_on_values,omitempty"`
}

type FilterSpec struct {
	RowLengthStrict bool            `json:"row_length_strict,omitzero"`
	When            *ExpressionNode `json:"when,omitzero"`
	MaxOutputCount  int             `json:"max_output_records,omitzero"`
}

type FilterTransformationPipe struct {
	// contains filtered or unexported fields
}

type FixedWidthColumn struct {
	Start      int
	End        int
	ColumnName string
}

type FixedWidthEncodingInfo struct {
	RecordTypeColumn *FixedWidthColumn
	ColumnsMap       map[string]*[]*FixedWidthColumn
	ColumnsOffsetMap map[string]int
	RecordTypeList   []string
}

type Float32Builder struct {
	// contains filtered or unexported fields
}

type Float64Builder struct {
	// contains filtered or unexported fields
}

type FormatDatePF struct {
	// contains filtered or unexported fields
}

type FunctionTokenNode struct {
	Type            string         `json:"type"`
	ParseDateConfig *ParseDateSpec `json:"parse_date_config,omitzero"`
	LargeDouble     *float64       `json:"large_double,omitzero"`
}

type GroupBySpec struct {
	GroupByName  []string `json:"group_by_name,omitempty"`
	GroupByPos   []int    `json:"group_by_pos,omitempty"`
	GroupByCount int      `json:"group_by_count,omitzero"`
	DomainKey    string   `json:"domain_key,omitempty"`
	IsDebug      bool     `json:"is_debug,omitzero"`
}

type GroupByTransformationPipe struct {
	// contains filtered or unexported fields
}

type HashEvaluator struct {
	// contains filtered or unexported fields
}

type HashExpression struct {
	Expr                   string   `json:"expr,omitempty"`
	CompositeExpr          []string `json:"composite_expr,omitempty"`
	DomainKey              string   `json:"domain_key,omitempty"`
	NbrJetsPartitionsAny   any      `json:"nbr_jets_partitions,omitzero"`
	MultiStepShardingMode  string   `json:"multi_step_sharding_mode,omitempty"`
	AlternateCompositeExpr []string `json:"alternate_composite_expr,omitempty"`
	NoPartitions           bool     `json:"no_partitions,omitzero"`
	ComputeDomainKey       bool     `json:"compute_domain_key,omitzero"`
}

type HashingAlgoEnum int

type HighFreqSpec struct {
	Name          string `json:"name"`
	KeyRe         string `json:"key_re,omitempty"`
	TopPercentile int    `json:"top_pct,omitzero"`
	TopRank       int    `json:"top_rank,omitzero"`
	// contains filtered or unexported fields
}

type HighFreqTransformationPipe struct {
	// contains filtered or unexported fields
}

type Input2PipeSet map[string]*PipeSet

type InputChannel struct {
	Name           string
	Channel        <-chan []any
	Columns        *map[string]int
	DomainKeySpec  *DomainKeysSpec
	Config         *ChannelSpec
	HasGroupedRows bool
}

type InputChannelConfig struct {
	// Type range: memory (default), input, stage
	// Format: csv, headerless_csv, etc.
	// ReadBatchSize: nbr of rows to read per record (format: parquet)
	// Compression: none, snappy (parquet: always snappy)
	// DetectEncoding: Detect file encoding (limited) for text file format
	// DetectCrAsEol: Detect if \r is used as eol (format: csv,headerless_csv)
	// DiscardFileHeaders: when true, discard the headers from the input file (typically for csv format).
	// EolByte: Byte to use as eol (format: csv,headerless_csv)
	// MultiColumnsInput: Indicate that input file must have multiple columns,
	// this is used to detect if the wrong delimiter is used (csv,headerless_csv)
	// Note: SchemaProvider, Compression, Format for Type input are provided via
	// ComputePipesCommonArgs.SourcesConfig (ie input_registry table).
	// BadRowsConfig: Specify how to handle bad rows.
	// HasGroupedRows indicates that the channel contains grouped rows,
	// most likely from the group_by or merge operator.
	// Note: The input_row channel (main input) will be cast to the
	// rdf type specified by the domain class of the main input source.
	FileConfig
	Type             string               `json:"type"`
	Name             string               `json:"name"`
	SchemaProvider   string               `json:"schema_provider,omitempty"`
	ReadSessionId    string               `json:"read_session_id,omitempty"`
	ReadStepId       string               `json:"read_step_id,omitempty"`
	ReadPartitionId  string               `json:"read_partition_id,omitempty"`
	SamplingRate     int                  `json:"sampling_rate,omitzero"`
	SamplingMaxCount int                  `json:"sampling_max_count,omitzero"`
	HasGroupedRows   bool                 `json:"has_grouped_rows,omitzero"`
	MergeChannels    []InputChannelConfig `json:"merge_channels,omitempty"`
	// contains filtered or unexported fields
}

type InputMappingExpr struct {
	InputColumn           sql.NullString
	DataProperty          string
	CleansingFunctionName sql.NullString
	Argument              sql.NullString
	DefaultValue          sql.NullString
	ErrorMessage          sql.NullString
}

type InputRowColumns struct {
	OriginalHeaders []string `json:"original_headers"`
	MainInput       []string `json:"main_input"`
}

type InputSourceSpec struct {
	OriginalInputColumns []string        `json:"original_input_columns,omitempty"`
	InputColumns         []string        `json:"input_columns,omitempty"`
	DomainClass          string          `json:"domain_class,omitempty"`
	DomainKeys           *DomainKeysSpec `json:"domain_keys_spec,omitempty"`
}

type Int32Builder struct {
	// contains filtered or unexported fields
}

type Int64Builder struct {
	// contains filtered or unexported fields
}

type JetRdfSession interface {
	GetResourceManager() JetResourceManager
	JetResources() *JetResources

	Insert(s, p RdfNode, o RdfNode) error
	Erase(s, p RdfNode, o RdfNode) (bool, error)
	Retract(s, p RdfNode, o RdfNode) (bool, error)
	Contains(s, p RdfNode, o RdfNode) bool
	ContainsSP(s, p RdfNode) bool
	GetObject(s, p RdfNode) RdfNode

	FindSPO(s, p, o RdfNode) TripleIterator
	FindSP(s, p RdfNode) TripleIterator
	FindS(s RdfNode) TripleIterator
	Find() TripleIterator

	NewReteSession(ruleset string) (JetReteSession, error)
	Release() error
}

type JetResourceManager interface {
	RdfNull() RdfNode
	CreateBNode(key int) RdfNode
	NewDateLiteral(data string) RdfNode
	NewDateDetails(year, month, day int) RdfNode
	NewDatetimeLiteral(data string) RdfNode
	NewDatetimeDetails(year, month, day, hour, min, sec int) RdfNode
	NewDoubleLiteral(x float64) RdfNode
	NewIntLiteral(data int) RdfNode
	NewUIntLiteral(data uint) RdfNode
	NewResource(name string) RdfNode
	NewTextLiteral(data string) RdfNode
}

type JetResources struct {
	Jets__client                  RdfNode
	Jets__completed               RdfNode
	Jets__currentSourcePeriod     RdfNode
	Jets__currentSourcePeriodDate RdfNode
	Jets__entity_property         RdfNode
	Jets__exception               RdfNode
	Jets__from                    RdfNode
	Jets__input_record            RdfNode
	Jets__istate                  RdfNode
	Jets__key                     RdfNode
	Jets__length                  RdfNode
	Jets__lookup_multi_rows       RdfNode
	Jets__lookup_row              RdfNode
	Jets__loop                    RdfNode
	Jets__max_vertex_visits       RdfNode
	Jets__operator                RdfNode
	Jets__org                     RdfNode
	Jets__range_value             RdfNode
	Jets__replace_chars           RdfNode
	Jets__replace_with            RdfNode
	Jets__source_period_sequence  RdfNode
	Jets__sourcePeriodType        RdfNode
	Jets__state                   RdfNode
	Jets__value_property          RdfNode
	Rdf__type                     RdfNode
}

type JetReteSession interface {
	ExecuteRules() error
	Release() error
}

type JetRuleEngine interface {
	MainRuleFile() string
	JetResources() *JetResources
	GetMetaGraphTriples() []string

	GetMetaResourceManager() JetResourceManager
	Insert(s, p RdfNode, o RdfNode) error

	NewRdfSession() (JetRdfSession, error)
	Release() error
}

type JetRulesFactory interface {
	JetRulesName() string
	NewJetRuleEngine(dbpool *pgxpool.Pool, processName string, isDebug bool) (JetRuleEngine, error)
	ClearCache() bool
}

type JetRulesProxy interface {
	GetDefaultFactory() JetRulesFactory
	GetGoFactory() JetRulesFactory
	GetNativeFactory() JetRulesFactory
}

type JetrulesOutputChan struct {
	ClassName        string
	ColumnEvaluators []TransformationColumnEvaluator
	OutputCh         *OutputChannel
}

type JetrulesSpec struct {
	ProcessName             string                `json:"process_name,omitempty"`
	UseJetRulesNative       bool                  `json:"use_jet_rules_native,omitzero"`
	UseJetRulesGo           bool                  `json:"use_jet_rules_go,omitzero"`
	InputRdfType            string                `json:"input_rdf_type,omitempty"`
	MaxInputCount           int                   `json:"max_input_count,omitzero"`
	PoolSize                int                   `json:"pool_size,omitzero"`
	MaxReteSessionsSaved    int                   `json:"max_rete_sessions_saved,omitzero"`
	MaxLooping              int                   `json:"max_looping,omitzero"`
	CurrentSourcePeriod     int                   `json:"current_source_period,omitzero"`
	CurrentSourcePeriodDate string                `json:"current_source_period_date,omitempty"`
	CurrentSourcePeriodType string                `json:"current_source_period_type,omitempty"`
	RuleConfig              []map[string]any      `json:"rule_config,omitempty"`
	MetadataInputSources    []CsvSourceSpec       `json:"metadata_input_sources,omitempty"`
	IsDebug                 bool                  `json:"is_debug,omitzero"`
	OutputChannels          []OutputChannelConfig `json:"output_channels,omitempty"`
}

type JetrulesTransformationPipe struct {
	// contains filtered or unexported fields
}

type JetrulesWorkerResult struct {
	ReteSessionCount int64
	ErrorsCount      int64
	Err              error
}

type JetsPartitionInfo struct {
	PartitionLabel string
	PartitionSize  int64
}

type JrPoolManager struct {
	WorkersTaskCh chan []interface{}

	WaitForDone *sync.WaitGroup
	// contains filtered or unexported fields
}

type JrPoolWorker struct {
	// contains filtered or unexported fields
}

type KeywordCount struct {
	Name     string
	Keywords []string
	Count    int
}

type KeywordTokenNode struct {
	Name     string   `json:"name"`
	Keywords []string `json:"keywords,omitempty"`
}

type LoadFromS3FilesResult struct {
	LoadRowCount int64
	BadRowCount  int64
	Err          error
}

type LookupColumnSpec struct {
	// Type range: select, value
	Name string  `json:"name,omitempty"`
	Type string  `json:"type,omitempty"`
	Expr *string `json:"expr,omitzero"`
}

type LookupCount struct {
	Name  string
	Count int
}

type LookupSpec struct {
	// type range: sql_lookup, s3_csv_lookup
	Key          string            `json:"key"`
	Type         string            `json:"type"`
	Query        string            `json:"query,omitempty"`     // for sql_lookup
	CsvSource    *CsvSourceSpec    `json:"csv_source,omitzero"` //for s3_csv_lookup
	Columns      []TableColumnSpec `json:"columns,omitempty"`
	LookupKey    []string          `json:"lookup_key,omitempty"`
	LookupValues []string          `json:"lookup_values,omitempty"`
}

type LookupTable interface {
	// Returns the lookup row associated with key
	Lookup(key *string) (*[]any, error)
	// Returns the row's value associated with the lookup column
	LookupValue(row *[]any, columnName string) (any, error)
	// Returns the mapping between column name to pos in the returned row
	ColumnMap() map[string]int
	// Return true if the table is empty, ColumnMap is empty as well
	IsEmptyTable() bool
	// Return the number of rows in the lookup table
	Size() int64
}

type LookupTableManager struct {
	LookupTableMap map[string]LookupTable
	// contains filtered or unexported fields
}

type LookupTableS3 struct {
	// contains filtered or unexported fields
}

type LookupTableSql struct {
	// contains filtered or unexported fields
}

type LookupTokenNode struct {
	Name             string            `json:"lookup_name"`
	KeyRe            string            `json:"key_re,omitempty"`
	Tokens           []string          `json:"tokens,omitempty"`
	MultiTokensMatch []MultiTokensNode `json:"multi_tokens_match,omitempty"`
}

type LookupTokensState struct {
	LookupTbl        LookupTable
	KeyRe            *regexp.Regexp
	LookupMatch      map[string]*LookupCount
	MultiTokensMatch []MultiTokensNode
}

type MapExpression struct {
	CleansingFunction string `json:"cleansing_function,omitempty"`
	Argument          string `json:"argument,omitempty"`
	Default           string `json:"default,omitempty"`
	ErrMsg            string `json:"err_msg,omitempty"`
	RdfType           string `json:"rdf_type,omitempty"`
}

type MapRecordSpec struct {
	FileMappingTableName string `json:"file_mapping_table_name"`
	IsDebug              bool   `json:"is_debug,omitzero"`
}

type MapRecordTransformationPipe struct {
	// contains filtered or unexported fields
}

type MergeCurrentValue struct {
	// contains filtered or unexported fields
}

type MergeFileReader struct {
	// contains filtered or unexported fields
}

type MergeFileSpec struct {
	FirstPartitionHasHeaders bool `json:"first_partition_has_headers,omitempty"` // splitter column
}

type MergeSpec struct {
	IsDebug      bool           `json:"is_debug,omitzero"`
	MainGroupBy  GroupBySpec    `json:"main_group_by"`
	MergeGroupBy []*GroupBySpec `json:"merge_group_by,omitempty"`
}

type MergeTransformationPipe struct {
	// contains filtered or unexported fields
}

type Metric struct {
	// Type range: runtime
	// Name values: alloc_mb, total_alloc_mb, sys_mb, nbr_gc
	// note: suffix _mb for units in MiB
	Type string `json:"type"`
	Name string `json:"name"`
}

type MetricsSpec struct {
	ReportInterval int      `json:"report_interval_sec"`
	RuntimeMetrics []Metric `json:"runtime_metrics"`
}

type MinMaxValue struct {
	MinValue   string
	MaxValue   string
	MinMaxType string
	HitRatio   float64
	NbrSamples int
}

type MultiTokensNode struct {
	Name      string   `json:"name"`
	NbrTokens int      `json:"nbr_tokens"`
	Tokens    []string `json:"tokens"`
	// contains filtered or unexported fields
}

type OutputChannel struct {
	Name    string
	Channel chan<- []any
	Columns *map[string]int
	Config  *ChannelSpec
}

type OutputChannelConfig struct {
	// Type range: memory (default), stage, output, sql
	// Format: csv, headerless_csv, etc.
	// NbrRowsInRecord: nbr of rows in record (format: parquet)
	// Compression: none, snappy (default).
	// UseInputParquetSchema to use the same schema as the input file.
	// UseOriginalHeaders to use the headers from the input file (csv only).
	// Must have save_parquet_schema = true in the cpipes first input_channel.
	// OutputLocation: jetstore_s3_input, jetstore_s3_output (default), or custom location.
	// When OutputLocation is jetstore_s3_input it will also write to the input bucket.
	// When OutputLocation uses a custom location, it replaces KeyPrefix and FileName.
	// OutputLocation must ends with "/" if we want to use default file name
	// (i.e. OutputLocation does not include the file name).
	// Note: refactoring using FileConfig.FileKey is synonym to OutputLocation
	// KeyPrefix is optional, default to $PATH_FILE_KEY.
	// Use $CURRENT_PARTITION_LABEL in KeyPrefix and FileName to substitute with
	// current partition label.
	// Other available env substitution:
	// $FILE_KEY main input file key.
	// $SESSIONID current session id.
	// $PROCESS_NAME current process name.
	// $PATH_FILE_KEY file key path portion.
	// $NAME_FILE_KEY file key file name portion (empty when in part files mode).
	// $SHARD_ID current node id.
	// $JETS_PARTITION_LABEL current node partition label.
	FileConfig
	Type                  string `json:"type"`
	Name                  string `json:"name,omitempty"`
	UseOriginalHeaders    bool   `json:"use_original_headers,omitzero"`     // Type output
	UseInputParquetSchema bool   `json:"use_input_parquet_schema,omitzero"` // Type stage,output
	SchemaProvider        string `json:"schema_provider,omitempty"`         // Type stage,output, alt to Format
	WriteStepId           string `json:"write_step_id,omitempty"`           // Type stage
	OutputTableKey        string `json:"output_table_key,omitempty"`        // Type sql
	FileKey2              string `json:"output_location,omitempty"`         // Type output
	SpecName              string `json:"channel_spec_name,omitempty"`
}