reader

type CSVReader struct {
	// InputFiles is a list of CSV file paths to read
	InputFiles []string
	// InputDir is a directory containing CSV files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// Delimiter is the field delimiter (default: comma)
	Delimiter rune
	// HasHeader indicates if the first row is a header row
	HasHeader bool
	// TextColumns are column names or indices to use as document text.
	// If empty, all columns are concatenated as text.
	TextColumns []string
	// MetadataColumns are column names or indices to extract as metadata.
	// If empty, all non-text columns are used as metadata.
	MetadataColumns []string
	// ConcatRows determines if all rows should be concatenated into a single document.
	// If false (default), each row becomes a separate document.
	ConcatRows bool
	// RowSeparator is used when ConcatRows is true (default: newline)
	RowSeparator string
}

type CSVStreamReader struct {
	*CSVReader
	// contains filtered or unexported fields
}

type DocxReader struct {
	// InputFiles is a list of DOCX file paths to read
	InputFiles []string
	// InputDir is a directory containing DOCX files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// ExtractImages determines if images should be extracted as separate nodes
	ExtractImages bool
	// PreserveParagraphs keeps paragraph breaks in the output
	PreserveParagraphs bool
	// ExtractMetadata extracts document properties (author, title, etc.)
	ExtractMetadata bool
	// ExtractTables extracts table content
	ExtractTables bool
}

type ExcelReader struct {
	// InputFiles is a list of Excel file paths to read
	InputFiles []string
	// InputDir is a directory containing Excel files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// SheetNames specifies which sheets to read. If empty, all sheets are read.
	SheetNames []string
	// HasHeader indicates if the first row is a header row
	HasHeader bool
	// TextColumns are column names or indices to use as document text.
	// If empty, all columns are concatenated as text.
	TextColumns []string
	// MetadataColumns are column names or indices to extract as metadata.
	// If empty, all non-text columns are used as metadata.
	MetadataColumns []string
	// ConcatRows determines if all rows should be concatenated into a single document.
	// If false (default), each row becomes a separate document.
	ConcatRows bool
	// ConcatSheets determines if all sheets should be concatenated into a single document.
	// If false (default), each sheet is processed separately.
	ConcatSheets bool
	// RowSeparator is used when ConcatRows is true (default: newline)
	RowSeparator string
	// SheetSeparator is used when ConcatSheets is true (default: double newline)
	SheetSeparator string
}

type FileReader interface {
	Reader
	// LoadFromFile loads a document from a specific file path.
	LoadFromFile(filePath string) ([]schema.Node, error)
}

type HTMLReader struct {
	// InputFiles is a list of HTML file paths to read
	InputFiles []string
	// InputDir is a directory containing HTML files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// TagsToExtract specifies which HTML tags to extract text from.
	// If empty, extracts from body. Common values: "p", "div", "article", "main"
	TagsToExtract []string
	// TagsToRemove specifies which HTML tags to remove entirely (e.g., "script", "style")
	TagsToRemove []string
	// PreserveWhitespace keeps original whitespace formatting
	PreserveWhitespace bool
}

type JSONReader struct {
	// InputFiles is a list of JSON file paths to read
	InputFiles []string
	// InputDir is a directory containing JSON files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// TextContentKey is the JSON key to use as document text content.
	// If empty, the entire JSON is serialized as text.
	TextContentKey string
	// MetadataKeys are JSON keys to extract as document metadata.
	// If empty, all non-text keys are used as metadata.
	MetadataKeys []string
	// IsJSONL indicates if files are JSON Lines format (one JSON object per line)
	IsJSONL bool
}

type LazyReader interface {
	Reader
	// LazyLoadData returns a channel that yields documents one at a time.
	// The channel is closed when all documents have been loaded or an error occurs.
	LazyLoadData() (<-chan schema.Node, <-chan error)
}

type MarkdownReader struct {
	// InputFiles is a list of Markdown file paths to read
	InputFiles []string
	// InputDir is a directory containing Markdown files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// RemoveHyperlinks removes hyperlinks from the text
	RemoveHyperlinks bool
	// RemoveImages removes image references from the text
	RemoveImages bool
	// SplitByHeaders splits document into multiple nodes by headers
	SplitByHeaders bool
	// HeadersToSplitOn specifies which header levels to split on (e.g., []int{1, 2})
	HeadersToSplitOn []int
}

type PDFReader struct {
	// InputFiles is a list of PDF file paths to read
	InputFiles []string
	// InputDir is a directory containing PDF files
	InputDir string
	// Recursive determines if subdirectories should be searched
	Recursive bool
	// SplitByPage creates separate nodes for each page
	SplitByPage bool
	// ExtraMetadata is additional metadata to add to all documents
	ExtraMetadata map[string]interface{}
	// PasswordFunc is a function that returns the password for a PDF file
	// The function receives the file path and should return the password
	PasswordFunc func(filePath string) string
}

type PDFReaderOption func(*PDFReader)

type Reader interface {
	// LoadData loads documents and returns them as a slice.
	LoadData() ([]schema.Node, error)
}

type ReaderError struct {
	Source  string // File path or URL that caused the error
	Message string
	Err     error
}

type ReaderMetadata struct {
	// Name is the reader name (e.g., "JSONReader", "PDFReader")
	Name string
	// SupportedExtensions lists file extensions this reader supports
	SupportedExtensions []string
	// Description describes what this reader does
	Description string
}

type ReaderOptions struct {
	// Recursive determines if directory readers should recurse into subdirectories
	Recursive bool
	// FileExtensions filters which file extensions to process
	FileExtensions []string
	// ExcludePatterns are glob patterns for files/dirs to exclude
	ExcludePatterns []string
	// IncludeHidden determines if hidden files should be included
	IncludeHidden bool
	// NumWorkers is the number of concurrent workers for parallel loading
	NumWorkers int
	// ExtraMetadata is additional metadata to add to all loaded documents
	ExtraMetadata map[string]interface{}
}

type ReaderWithContext interface {
	Reader
	// LoadDataWithContext loads documents with context support.
	LoadDataWithContext(ctx context.Context) ([]schema.Node, error)
}

type ReaderWithMetadata interface {
	Reader
	// Metadata returns information about this reader.
	Metadata() ReaderMetadata
}

type SimpleDirectoryReader struct {
	// contains filtered or unexported fields
}