thunder

package module
v0.0.0-...-bad0887 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 21, 2024 License: MIT Imports: 15 Imported by: 9

README ΒΆ

Thunder

Container Registry Static Assets Documentation License

Thunder is a lightweight tool designed to easily synchronize your SQL databases (Postgres, MySQL, etc.) with search engines (ElasticSearch, OpenSearch, etc.) without impacting your database's performance.

It is optimized for real-time updates, ensuring that your data is always indexed and searchable.

Notes

This library is under active development.

Features and APIs may change.

Contributions and feedback are welcome!

Features

  • Non-Intrusive Synchronization: Sync your SQL data without impacting your database’s performance.
  • Real-Time Indexing: Automatically update indexes as soon as your data changes.
  • Scalable: Designed to handle databases of all sizes, ensuring fast and efficient syncs.

🌐 Visit Our Website

For more information, updates, and resources, check out the official website:

πŸ“š Documentation

Our detailed documentation is available to help you get started, learn how to configure and use Thunder, and explore advanced features:

Contributing

  1. Fork the repository.
  2. Create a new branch for your feature or bugfix.
  3. Commit your changes.
  4. Push your branch.
  5. Create a pull request.

Credits

License

MIT. See the License File for more information.

Documentation ΒΆ

Index ΒΆ

Constants ΒΆ

This section is empty.

Variables ΒΆ

View Source
var Exporters = utils.NewRegistry[Exporter]("exporter")

Exporters is a registry that allows external library to register their own exporter

View Source
var Modules = utils.NewRegistry[Module]("module").ValidateUsing(func(ID string, module Module) error {
	if module.New == nil {
		return errors.New(`module doesn't implements "New" method`)
	}
	if val := module.New(); val == nil {
		return errors.New(`module "New" method must return a non-nil instance`)
	}
	return nil
})

Modules is a registry that allows external library to register their own modules

View Source
var Processors = utils.NewRegistry[*Processor]("processor").SetIdGenerator(func(item **Processor) (string, error) {
	ulid := ulid.Make().String()
	(*item).ID = ulid
	return ulid, nil
})
View Source
var SourceDrivers = utils.NewRegistry[SourceDriver]("source-driver").ValidateUsing(func(ID string, driver SourceDriver) error {
	if driver.New == nil {
		return errors.New(`target driver doesn't implements "New" method`)
	}
	return nil
})

SourceDrivers is a registry that allows external library to register their own source driver

View Source
var Sources = utils.NewRegistry[Source]("source").SetIdGenerator(func(item *Source) (string, error) {
	ulid := ulid.Make().String()
	item.ID = ulid
	return ulid, nil
})
View Source
var TargetDrivers = utils.NewRegistry[TargetDriver]("target-driver").ValidateUsing(func(ID string, driver TargetDriver) error {
	if driver.New == nil {
		return errors.New(`target driver doesn't implements "New" method`)
	}
	return nil
})

TargetDrivers is a registry that allows external library to register their own target driver

View Source
var Targets = utils.NewRegistry[Target]("target").SetIdGenerator(func(item *Target) (string, error) {
	ulid := ulid.Make().String()
	item.ID = ulid
	return ulid, nil
})

Functions ΒΆ

func GetConfigPath ΒΆ

func GetConfigPath() string

func GetLoggerForModule ΒΆ

func GetLoggerForModule(ModuleID string) *zerolog.Logger

func GetLoggerForProcessor ΒΆ

func GetLoggerForProcessor(processor *Processor) *zerolog.Logger

func GetLoggerForSourceDriver ΒΆ

func GetLoggerForSourceDriver(driverID string) *zerolog.Logger

func GetLoggerForTargetDriver ΒΆ

func GetLoggerForTargetDriver(driver TargetDriver) *zerolog.Logger

func LoadConfig ΒΆ

func LoadConfig() error

func SaveConfig ΒΆ

func SaveConfig() error

func SetConfigPath ΒΆ

func SetConfigPath(newPath string)

func Start ΒΆ

func Start() error

Types ΒΆ

type App ΒΆ

type App struct {
	// contains filtered or unexported fields
}

func GetApp ΒΆ

func GetApp() *App

type Condition ΒΆ

type Condition struct {
	Column   string `json:"column"`
	Operator string `json:"operator"`
	Value    string `json:"value,omitempty"`
}

type DbDeleteEvent ΒΆ

type DbDeleteEvent struct {
	Relation *Relation
	Pkey     string
}

type DbEvent ΒΆ

type DbEvent any // DbDeleteEvent | DbInsertEvent | DbPatchEvent | DbTruncateEvent

type DbInsertEvent ΒΆ

type DbInsertEvent struct {
	Pkey    string
	Version int
}

type DbPatchEvent ΒΆ

type DbPatchEvent struct {
	Relation  *Relation
	Version   int
	Pkey      string
	JsonPatch []byte
}

type DbTruncateEvent ΒΆ

type DbTruncateEvent struct {
	Relation *Relation
}

type Document ΒΆ

type Document struct {
	Pkey    string `json:"_pkey"`
	Version int    `json:"_version"`

	Json []byte `json:"json"`
}

type Exporter ΒΆ

type Exporter interface {
	Name() string
	MimeType() string
	Load(w io.Writer) error
	BeforeAll() error
	WriteDocument(doc *Document, itemPosition uint64) error
	AfterAll() error
}

Exporter is an interface for transforming a set of documents into a file.

type JsonConfig ΒΆ

type JsonConfig struct {
	Sources    []*JsonSource    `json:"sources"`
	Targets    []*JsonTarget    `json:"targets"`
	Processors []*JsonProcessor `json:"processors"`
}

type JsonMapping ΒΆ

type JsonMapping []JsonMappingField

func SerializeMapping ΒΆ

func SerializeMapping(m *Mapping) (*JsonMapping, error)

type JsonMappingField ΒΆ

type JsonMappingField struct {
	FieldType string `json:"_type"`

	// Simple field
	Column string `json:"column,omitempty"`
	Alias  string `json:"alias,omitempty"`

	// Relation
	Name        string   `json:"name,omitempty"`
	Type        string   `json:"type,omitempty"`
	Table       string   `json:"table,omitempty"`
	PrimaryKeys []string `json:"primary_keys,omitempty"`
	LocalKey    string   `json:"local_key,omitempty"`

	UsePivotTable   bool               `json:"use_pivot_table,omitempty"`
	PivotTable      string             `json:"pivot_table,omitempty"`
	ForeignPivotKey string             `json:"foreign_pivot_key,omitempty"`
	LocalPivotKey   string             `json:"local_pivot_key,omitempty"`
	PivotFields     []JsonMappingField `json:"pivot_fields,omitempty"`

	ForeignKey string `json:"foreign_key,omitempty"`

	Mapping JsonMapping `json:"mapping,omitempty"`
}

type JsonProcessor ΒΆ

type JsonProcessor struct {
	ID string `json:"ID"`

	Source  string   `json:"source"`  // as source_id
	Targets []string `json:"targets"` // as targets_id

	Table       string      `json:"table"`
	PrimaryKeys []string    `json:"primary_keys"`
	Conditions  []Condition `json:"conditions"`

	Mapping JsonMapping `json:"mapping"`

	Index string `json:"index"`

	Enabled bool `json:"enabled"`
}

func SerializeProcessor ΒΆ

func SerializeProcessor(p *Processor) (*JsonProcessor, error)

type JsonSource ΒΆ

type JsonSource struct {
	ID     string `json:"id"`
	Driver string `json:"driver"`
	Config any    `json:"config"`
}

func SerializeSource ΒΆ

func SerializeSource(s *Source) (*JsonSource, error)

type JsonTarget ΒΆ

type JsonTarget struct {
	ID     string `json:"id"`
	Driver string `json:"driver"`
	Config any    `json:"config"`
}

func SerializeTarget ΒΆ

func SerializeTarget(t *Target) (*JsonTarget, error)

type Mapping ΒΆ

type Mapping struct {
	Fields    []SimpleField
	Relations []Relation
}

func UnserializeMapping ΒΆ

func UnserializeMapping(jm *JsonMapping, parent *Relation) (*Mapping, error)

type Module ΒΆ

type Module interface {
	RequiredModules() []string

	New() Module

	Start() error
	Stop() error
}

type Processor ΒΆ

type Processor struct {
	ID string

	Indexing atomic.Bool

	Listening         atomic.Bool
	ListenBroadcaster *utils.Broadcaster[DbEvent, TargetEvent]

	Source  *Source
	Targets []*Target

	Table       string
	PrimaryKeys []string
	Conditions  []Condition

	Mapping Mapping

	Index string

	Enabled bool
}

func UnserializeProcessor ΒΆ

func UnserializeProcessor(jp *JsonProcessor) (*Processor, error)

func (*Processor) FullIndex ΒΆ

func (p *Processor) FullIndex(ctx context.Context) error

func (*Processor) Start ΒΆ

func (p *Processor) Start() error

func (*Processor) Stop ΒΆ

func (p *Processor) Stop() error

func (*Processor) StreamDocuments ΒΆ

func (p *Processor) StreamDocuments(ctx context.Context, exporter Exporter, w io.Writer, limit uint64) error

type Relation ΒΆ

type Relation struct {
	Name string

	Many bool

	Table       string
	PrimaryKeys []string

	LocalKey   string
	Pivot      *RelationPivot
	ForeignKey string

	Mapping Mapping

	// Recursive navigation
	Parent   *Relation
	Children []*Relation
}

func (*Relation) Path ΒΆ

func (r *Relation) Path() string

type RelationPivot ΒΆ

type RelationPivot struct {
	Table      string
	LocalKey   string
	ForeignKey string

	Fields []SimpleField
}

type SimpleField ΒΆ

type SimpleField struct {
	Column string
	Name   *string // Use Column as Name if empty or nil
}

type Source ΒΆ

type Source struct {
	ID     string
	Driver SourceDriver
	Config utils.DynamicConfig
}

func UnserializeSource ΒΆ

func UnserializeSource(js *JsonSource) (*Source, error)

type SourceDriver ΒΆ

type SourceDriver interface {
	ID() string

	New(config any) (SourceDriver, error)

	Config() SourceDriverConfig

	TestConfig() (string, error) // TODO USELESS REPLACE IN FAVOR OF STATS TO CHECK NOT EMPTY
	Stats() (*SourceDriverStats, error)

	// GetDocumentsForProcessor use limit=0 to unlimited
	GetDocumentsForProcessor(processor *Processor, in chan<- *Document, ctx context.Context, limit uint64) error

	Start(processor *Processor, in utils.BroadcasterIn[DbEvent]) error
	Stop() error
}

type SourceDriverConfig ΒΆ

type SourceDriverConfig struct {
	Name   string              `json:"name"`
	Config utils.DynamicConfig `json:"-"`

	// As inlined SVG
	Image string   `json:"image,omitempty"`
	Notes []string `json:"notes,omitempty"`
}

type SourceDriverStats ΒΆ

type SourceDriverStats map[string]SourceDriverStatsTable

type SourceDriverStatsTable ΒΆ

type SourceDriverStatsTable struct {
	Columns     []string `json:"columns"`
	PrimaryKeys []string `json:"primary_keys"`
}

type Target ΒΆ

type Target struct {
	ID     string
	Driver TargetDriver
	Config utils.DynamicConfig
}

func UnserializeTarget ΒΆ

func UnserializeTarget(js *JsonTarget) (*Target, error)

type TargetDeleteEvent ΒΆ

type TargetDeleteEvent struct {
	Relation *Relation
	Pkey     string
}

type TargetDriver ΒΆ

type TargetDriver interface {
	ID() string

	New(config any) (TargetDriver, error)

	Config() TargetDriverConfig

	TestConfig() (string, error) // TODO USELESS REPLACE IN FAVOR OF STATS TO CHECK NOT EMPTY

	HandleEvents(processor *Processor, eventsChan <-chan TargetEvent) error

	Shutdown() error
}

type TargetDriverConfig ΒΆ

type TargetDriverConfig struct {
	Name   string              `json:"name"`
	Config utils.DynamicConfig `json:"-"`

	// As inlined SVG
	Image string   `json:"image,omitempty"`
	Notes []string `json:"notes,omitempty"`
}

type TargetEvent ΒΆ

type TargetEvent any // TargetDeleteEvent | TargetInsertEvent | TargetPatchEvent | TargetTruncateEvent

type TargetInsertEvent ΒΆ

type TargetInsertEvent struct {
	Pkey    string
	Version int
	Json    []byte
}

type TargetPatchEvent ΒΆ

type TargetPatchEvent struct {
	Relation  *Relation
	Pkey      string
	Version   int
	JsonPatch []byte
}

type TargetTruncateEvent ΒΆ

type TargetTruncateEvent struct {
	Relation *Relation
}

Directories ΒΆ

Path Synopsis
app module
exporters
csv
modules
api
source-drivers
target-drivers

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL