Skip to content

Commit

Permalink
Update gpt4all bindings, fix search path on new versions
Browse files Browse the repository at this point in the history
  • Loading branch information
@mudler
mudler committed Jun 14, 2023
1 parent 2328bba commit 20250dc
Show file tree
Hide file tree
Showing 8 changed files with 50 additions and 51 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ BINARY_NAME=local-ai

GOLLAMA_VERSION?=5f1620443a59c5531b5a15a16cd68f600a8437e9
GPT4ALL_REPO?=https://github.com/go-skynet/gpt4all
GPT4ALL_VERSION?=f7498c9
GPT4ALL_VERSION?=d34c513e01174fe83c6042403a0d183e56478d56
GOGGMLTRANSFORMERS_VERSION?=01b8436f44294d0e1267430f9eda4460458cec54
RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp
RWKV_VERSION?=1e18b2490e7e32f6b00e16f6a9ec0dd3a3d09266
Expand Down
6 changes: 6 additions & 0 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package api
import (
"errors"

"github.com/go-skynet/LocalAI/pkg/assets"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gofiber/fiber/v2/middleware/logger"
Expand Down Expand Up @@ -68,6 +69,11 @@ func App(opts ...AppOption) (*fiber.App, error) {
}

if options.assetsDestination != "" {
// Extract files from the embedded FS
err := assets.ExtractFiles(options.backendAssets, options.assetsDestination)
if err != nil {
log.Warn().Msgf("Failed extracting backend assets files: %s (might be required for some backends to work properly, like gpt4all)", err)
}
if err := PrepareBackendAssets(options.backendAssets, options.assetsDestination); err != nil {
log.Warn().Msgf("Failed extracting backend assets files: %s (might be required for some backends to work properly, like gpt4all)", err)
}
Expand Down
27 changes: 0 additions & 27 deletions api/backend_assets.go
View file

This file was deleted.

18 changes: 9 additions & 9 deletions api/openai.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func defaultRequest(modelFile string) OpenAIRequest {
// https://platform.openai.com/docs/api-reference/completions
func completionEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {
process := func(s string, req *OpenAIRequest, config *Config, loader *model.ModelLoader, responses chan OpenAIResponse) {
ComputeChoices(s, req, config, loader, func(s string, c *[]Choice) {}, func(s string) bool {
ComputeChoices(s, req, config, o, loader, func(s string, c *[]Choice) {}, func(s string) bool {
resp := OpenAIResponse{
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
Choices: []Choice{{Text: s}},
Expand Down Expand Up @@ -249,7 +249,7 @@ func completionEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {
log.Debug().Msgf("Template found, input modified to: %s", i)
}

r, err := ComputeChoices(i, input, config, o.loader, func(s string, c *[]Choice) {
r, err := ComputeChoices(i, input, config, o, o.loader, func(s string, c *[]Choice) {
*c = append(*c, Choice{Text: s})
}, nil)
if err != nil {
Expand Down Expand Up @@ -291,7 +291,7 @@ func embeddingsEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {

for i, s := range config.InputToken {
// get the model function to call for the result
embedFn, err := ModelEmbedding("", s, o.loader, *config)
embedFn, err := ModelEmbedding("", s, o.loader, *config, o)
if err != nil {
return err
}
Expand All @@ -305,7 +305,7 @@ func embeddingsEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {

for i, s := range config.InputStrings {
// get the model function to call for the result
embedFn, err := ModelEmbedding(s, []int{}, o.loader, *config)
embedFn, err := ModelEmbedding(s, []int{}, o.loader, *config, o)
if err != nil {
return err
}
Expand Down Expand Up @@ -341,7 +341,7 @@ func chatEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {
}
responses <- initialMessage

ComputeChoices(s, req, config, loader, func(s string, c *[]Choice) {}, func(s string) bool {
ComputeChoices(s, req, config, o, loader, func(s string, c *[]Choice) {}, func(s string) bool {
resp := OpenAIResponse{
Model: req.Model, // we have to return what the user sent here, due to OpenAI spec.
Choices: []Choice{{Delta: &Message{Content: s}}},
Expand Down Expand Up @@ -439,7 +439,7 @@ func chatEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {
return nil
}

result, err := ComputeChoices(predInput, input, config, o.loader, func(s string, c *[]Choice) {
result, err := ComputeChoices(predInput, input, config, o, o.loader, func(s string, c *[]Choice) {
*c = append(*c, Choice{Message: &Message{Role: "assistant", Content: s}})
}, nil)
if err != nil {
Expand Down Expand Up @@ -491,7 +491,7 @@ func editEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {
log.Debug().Msgf("Template found, input modified to: %s", i)
}

r, err := ComputeChoices(i, input, config, o.loader, func(s string, c *[]Choice) {
r, err := ComputeChoices(i, input, config, o, o.loader, func(s string, c *[]Choice) {
*c = append(*c, Choice{Text: s})
}, nil)
if err != nil {
Expand Down Expand Up @@ -616,7 +616,7 @@ func imageEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {

baseURL := c.BaseURL()

fn, err := ImageGeneration(height, width, mode, step, input.Seed, positive_prompt, negative_prompt, output, o.loader, *config)
fn, err := ImageGeneration(height, width, mode, step, input.Seed, positive_prompt, negative_prompt, output, o.loader, *config, o)
if err != nil {
return err
}
Expand Down Expand Up @@ -697,7 +697,7 @@ func transcriptEndpoint(cm *ConfigMerger, o *Option) func(c *fiber.Ctx) error {

log.Debug().Msgf("Audio file copied to: %+v", dst)

whisperModel, err := o.loader.BackendLoader(model.WhisperBackend, config.Model, []llama.ModelOption{}, uint32(config.Threads))
whisperModel, err := o.loader.BackendLoader(model.WhisperBackend, config.Model, []llama.ModelOption{}, uint32(config.Threads), o.assetsDestination)
if err != nil {
return err
}
Expand Down
20 changes: 10 additions & 10 deletions api/prediction.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
return llamaOpts
}

func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, dst string, loader *model.ModelLoader, c Config) (func() error, error) {
func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, dst string, loader *model.ModelLoader, c Config, o *Option) (func() error, error) {
if c.Backend != model.StableDiffusionBackend {
return nil, fmt.Errorf("endpoint only working with stablediffusion models")
}
inferenceModel, err := loader.BackendLoader(c.Backend, c.ImageGenerationAssets, []llama.ModelOption{}, uint32(c.Threads))
inferenceModel, err := loader.BackendLoader(c.Backend, c.ImageGenerationAssets, []llama.ModelOption{}, uint32(c.Threads), o.assetsDestination)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -88,7 +88,7 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
}, nil
}

func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, c Config) (func() ([]float32, error), error) {
func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, c Config, o *Option) (func() ([]float32, error), error) {
if !c.Embeddings {
return nil, fmt.Errorf("endpoint disabled for this model by API configuration")
}
Expand All @@ -100,9 +100,9 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, c Config)
var inferenceModel interface{}
var err error
if c.Backend == "" {
inferenceModel, err = loader.GreedyLoader(modelFile, llamaOpts, uint32(c.Threads))
inferenceModel, err = loader.GreedyLoader(modelFile, llamaOpts, uint32(c.Threads), o.assetsDestination)
} else {
inferenceModel, err = loader.BackendLoader(c.Backend, modelFile, llamaOpts, uint32(c.Threads))
inferenceModel, err = loader.BackendLoader(c.Backend, modelFile, llamaOpts, uint32(c.Threads), o.assetsDestination)
}
if err != nil {
return nil, err
Expand Down Expand Up @@ -240,7 +240,7 @@ func buildLLamaPredictOptions(c Config, modelPath string) []llama.PredictOption
return predictOptions
}

func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback func(string) bool) (func() (string, error), error) {
func ModelInference(s string, loader *model.ModelLoader, c Config, o *Option, tokenCallback func(string) bool) (func() (string, error), error) {
supportStreams := false
modelFile := c.Model

Expand All @@ -249,9 +249,9 @@ func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback
var inferenceModel interface{}
var err error
if c.Backend == "" {
inferenceModel, err = loader.GreedyLoader(modelFile, llamaOpts, uint32(c.Threads))
inferenceModel, err = loader.GreedyLoader(modelFile, llamaOpts, uint32(c.Threads), o.assetsDestination)
} else {
inferenceModel, err = loader.BackendLoader(c.Backend, modelFile, llamaOpts, uint32(c.Threads))
inferenceModel, err = loader.BackendLoader(c.Backend, modelFile, llamaOpts, uint32(c.Threads), o.assetsDestination)
}
if err != nil {
return nil, err
Expand Down Expand Up @@ -579,7 +579,7 @@ func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback
}, nil
}

func ComputeChoices(predInput string, input *OpenAIRequest, config *Config, loader *model.ModelLoader, cb func(string, *[]Choice), tokenCallback func(string) bool) ([]Choice, error) {
func ComputeChoices(predInput string, input *OpenAIRequest, config *Config, o *Option, loader *model.ModelLoader, cb func(string, *[]Choice), tokenCallback func(string) bool) ([]Choice, error) {
result := []Choice{}

n := input.N
Expand All @@ -589,7 +589,7 @@ func ComputeChoices(predInput string, input *OpenAIRequest, config *Config, load
}

// get the model function to call for the result
predFunc, err := ModelInference(predInput, loader, *config, tokenCallback)
predFunc, err := ModelInference(predInput, loader, *config, o, tokenCallback)
if err != nil {
return result, err
}
Expand Down
4 changes: 4 additions & 0 deletions examples/flowise/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,7 @@ docker-compose up --pull always

Open http://localhost:3000.

## Using LocalAI

Search for LocalAI in the integration, and use the `http://api:8080/` as URL.

16 changes: 16 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,19 @@ require (
golang.org/x/text v0.9.0 // indirect
golang.org/x/tools v0.9.3 // indirect
)

replace github.com/go-skynet/go-llama.cpp => /home/mudler/_git/LocalAI/go-llama

replace github.com/nomic-ai/gpt4all/gpt4all-bindings/golang => /home/mudler/_git/LocalAI/gpt4all/gpt4all-bindings/golang

replace github.com/go-skynet/go-ggml-transformers.cpp => /home/mudler/_git/LocalAI/go-ggml-transformers

replace github.com/donomii/go-rwkv.cpp => /home/mudler/_git/LocalAI/go-rwkv

replace github.com/ggerganov/whisper.cpp => /home/mudler/_git/LocalAI/whisper.cpp

replace github.com/go-skynet/go-bert.cpp => /home/mudler/_git/LocalAI/go-bert

replace github.com/go-skynet/bloomz.cpp => /home/mudler/_git/LocalAI/bloomz

replace github.com/mudler/go-stable-diffusion => /home/mudler/_git/LocalAI/go-stable-diffusion
8 changes: 4 additions & 4 deletions pkg/model/initializers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func rwkvLM(tokenFile string, threads uint32) func(string) (interface{}, error)
}
}

func (ml *ModelLoader) BackendLoader(backendString string, modelFile string, llamaOpts []llama.ModelOption, threads uint32) (model interface{}, err error) {
func (ml *ModelLoader) BackendLoader(backendString string, modelFile string, llamaOpts []llama.ModelOption, threads uint32, assetDir string) (model interface{}, err error) {
log.Debug().Msgf("Loading model %s from %s", backendString, modelFile)
switch strings.ToLower(backendString) {
case LlamaBackend:
Expand All @@ -161,7 +161,7 @@ func (ml *ModelLoader) BackendLoader(backendString string, modelFile string, lla
case StarcoderBackend:
return ml.LoadModel(modelFile, starCoder)
case Gpt4AllLlamaBackend, Gpt4AllMptBackend, Gpt4AllJBackend, Gpt4All:
return ml.LoadModel(modelFile, gpt4allLM(gpt4all.SetThreads(int(threads))))
return ml.LoadModel(modelFile, gpt4allLM(gpt4all.SetThreads(int(threads)), gpt4all.SetLibrarySearchPath(filepath.Join(assetDir, "backend-assets", "gpt4all"))))
case BertEmbeddingsBackend:
return ml.LoadModel(modelFile, bertEmbeddings)
case RwkvBackend:
Expand All @@ -175,7 +175,7 @@ func (ml *ModelLoader) BackendLoader(backendString string, modelFile string, lla
}
}

func (ml *ModelLoader) GreedyLoader(modelFile string, llamaOpts []llama.ModelOption, threads uint32) (interface{}, error) {
func (ml *ModelLoader) GreedyLoader(modelFile string, llamaOpts []llama.ModelOption, threads uint32, assetDir string) (interface{}, error) {
log.Debug().Msgf("Loading model '%s' greedly", modelFile)

ml.mu.Lock()
Expand All @@ -193,7 +193,7 @@ func (ml *ModelLoader) GreedyLoader(modelFile string, llamaOpts []llama.ModelOpt
continue
}
log.Debug().Msgf("[%s] Attempting to load", b)
model, modelerr := ml.BackendLoader(b, modelFile, llamaOpts, threads)
model, modelerr := ml.BackendLoader(b, modelFile, llamaOpts, threads, assetDir)
if modelerr == nil && model != nil {
log.Debug().Msgf("[%s] Loads OK", b)
return model, nil
Expand Down

0 comments on commit 20250dc

Please sign in to comment.