@@ -307,6 +307,7 @@ Model Choice:
+
Claude Compatibility Rename Fix
@@ -394,7 +395,7 @@ Unsaved data will be lost.
-
Top p Sampling ?Used to discard unlikely text in the sampling process. Lower values will make text more predictable but can become repetitious.
+
Top p Sampling ?Used to discard unlikely text in the sampling process. Lower values will make text more predictable but can become repetitious. Set to 1 to deactivate it.
diff --git a/koboldcpp.py b/koboldcpp.py
index 307533a55c69f..0b623863669bb 100755
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -296,7 +296,7 @@ def utfprint(str):
maxhordelen = 256
modelbusy = threading.Lock()
defaultport = 5001
-KcppVersion = "1.37"
+KcppVersion = "1.37.1"
showdebug = True
showsamplerwarning = True
exitcounter = 0
@@ -1391,11 +1391,14 @@ def onDropdownChange(event):
def run_horde_worker(args, api_key, worker_name):
import urllib.request
global friendlymodelname, maxhordectx, maxhordelen, exitcounter, modelbusy
+ epurl = f"http://localhost:{args.port}"
+ if args.host!="":
+ epurl = f"http://{args.host}:{args.port}"
def make_url_request(url, data, method='POST'):
try:
request = None
- headers = {"apikey": api_key,'User-Agent':'KoboldCpp Embedded Worker v1'}
+ headers = {"apikey": api_key,'User-Agent':'KoboldCpp Embedded Worker v1','Client-Agent':'KoboldCppEmbedWorker:1'}
if method=='POST':
json_payload = json.dumps(data).encode('utf-8')
request = urllib.request.Request(url, data=json_payload, headers=headers, method=method)
@@ -1427,7 +1430,7 @@ def make_url_request(url, data, method='POST'):
cluster = "https://horde.koboldai.net"
while exitcounter < 10:
time.sleep(2)
- readygo = make_url_request(f'http://localhost:{args.port}/api/v1/info/version', None,'GET')
+ readygo = make_url_request(f'{epurl}/api/v1/info/version', None,'GET')
if readygo:
print("Embedded Horde Worker is started.")
break
@@ -1472,7 +1475,7 @@ def make_url_request(url, data, method='POST'):
#do gen
while exitcounter < 10:
if not modelbusy.locked():
- current_generation = make_url_request(f'http://localhost:{args.port}/api/v1/generate', current_payload)
+ current_generation = make_url_request(f'{epurl}/api/v1/generate', current_payload)
if current_generation:
break
else:
diff --git a/llama.cpp b/llama.cpp
index 82f67a95d2e52..7ec893415eb33 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -187,6 +187,7 @@ struct llama_hparams {
// LLaMAv2
// TODO: load from model data hparams
float f_ffn_mult = 1.0f;
+ float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS;
float rope_freq_base = 10000.0f;
float rope_freq_scale = 1.0f;
@@ -870,6 +871,7 @@ struct llama_context_params llama_context_default_params() {
/*.n_ctx =*/ 512,
/*.n_batch =*/ 512,
/*.n_gqa =*/ 1,
+ /*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS,
/*.gpu_layers =*/ 0,
/*.main_gpu =*/ 0,
/*.tensor_split =*/ nullptr,
@@ -1001,6 +1003,7 @@ static void llama_model_load_internal(
int n_ctx,
int n_batch,
int n_gqa,
+ float rms_norm_eps,
int n_gpu_layers,
int main_gpu,
const float * tensor_split,
@@ -1026,6 +1029,9 @@ static void llama_model_load_internal(
auto & hparams = model.hparams;
+ // TODO: read from file
+ hparams.f_rms_norm_eps = rms_norm_eps;
+
{
switch (hparams.n_layer) {
case 26: model.type = e_model::MODEL_3B; break;
@@ -1079,6 +1085,7 @@ static void llama_model_load_internal(
fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer);
fprintf(stderr, "%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim
fprintf(stderr, "%s: n_gqa = %u\n", __func__, hparams.n_gqa());
+ fprintf(stderr, "%s: rnorm_eps = %.1e\n", __func__, hparams.f_rms_norm_eps);
fprintf(stderr, "%s: n_ff = %u\n", __func__, n_ff);
fprintf(stderr, "%s: freq_base = %.1f\n", __func__, hparams.rope_freq_base);
fprintf(stderr, "%s: freq_scale = %g\n", __func__, hparams.rope_freq_scale);
@@ -1351,6 +1358,7 @@ static bool llama_model_load(
int n_ctx,
int n_batch,
int n_gqa,
+ float rms_norm_eps,
int n_gpu_layers,
int main_gpu,
const float * tensor_split,
@@ -1364,7 +1372,7 @@ static bool llama_model_load(
llama_progress_callback progress_callback,
void *progress_callback_user_data) {
try {
- llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gqa, n_gpu_layers, main_gpu, tensor_split, rope_freq_base, rope_freq_scale, low_vram, memory_type,
+ llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gqa, rms_norm_eps, n_gpu_layers, main_gpu, tensor_split, rope_freq_base, rope_freq_scale, low_vram, memory_type,
use_mmap, use_mlock, vocab_only, progress_callback, progress_callback_user_data);
return true;
} catch (const std::exception & err) {
@@ -1417,10 +1425,12 @@ static bool llama_eval_internal(
const int64_t n_vocab = hparams.n_vocab;
const int64_t n_embd_gqa = hparams.n_embd_gqa();
+
LLAMA_ASSERT(n_embd_head == hparams.n_rot);
const float freq_base = hparams.rope_freq_base;
const float freq_scale = hparams.rope_freq_scale;
+ const float rms_norm_eps = hparams.f_rms_norm_eps;
const int n_gpu_layers = model.n_gpu_layers;
@@ -1500,7 +1510,7 @@ static bool llama_eval_internal(
// norm
{
- cur = ggml_rms_norm(ctx0, inpL);
+ cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps);
offload_func(cur);
ggml_set_name(cur, "rms_norm_0");
@@ -1648,7 +1658,7 @@ static bool llama_eval_internal(
{
// norm
{
- cur = ggml_rms_norm(ctx0, inpFF);
+ cur = ggml_rms_norm(ctx0, inpFF, rms_norm_eps);
offload_func(cur);
ggml_set_name(cur, "rms_norm_1");
@@ -1701,7 +1711,7 @@ static bool llama_eval_internal(
// norm
{
- cur = ggml_rms_norm(ctx0, inpL);
+ cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps);
offload_func_nr(cur);
ggml_set_name(cur, "rms_norm_2");
@@ -1725,12 +1735,17 @@ static bool llama_eval_internal(
// run the computation
ggml_build_forward_expand(&gf, cur);
+ // fprintf(stderr, "graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf.n_nodes, gf.n_leafs);
+
#if GGML_USE_MPI
ggml_mpi_graph_compute_pre(lctx.ctx_mpi, &gf, n_layer);
#endif
#ifdef GGML_USE_METAL
if (lctx.ctx_metal && N == 1) {
+ if (!ggml_metal_if_optimized(lctx.ctx_metal)) {
+ ggml_metal_graph_find_concurrency(lctx.ctx_metal,&gf);
+ }
ggml_metal_set_n_cb (lctx.ctx_metal, n_threads);
ggml_metal_graph_compute(lctx.ctx_metal, &gf);
ggml_metal_get_tensor (lctx.ctx_metal, cur);
@@ -3105,7 +3120,7 @@ struct llama_model * llama_load_model_from_file(
ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32;
- if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gqa, params.n_gpu_layers,
+ if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gqa, params.rms_norm_eps, params.n_gpu_layers,
params.main_gpu, params.tensor_split, params.rope_freq_base, params.rope_freq_scale,params.low_vram,
memory_type, params.use_mmap, params.use_mlock, params.vocab_only, params.progress_callback,
params.progress_callback_user_data)) {
diff --git a/llama.h b/llama.h
index 81a30e16b2a30..df46f9b9cfa31 100644
--- a/llama.h
+++ b/llama.h
@@ -53,6 +53,10 @@
#define LLAMA_SUPPORTS_GPU_OFFLOAD
#endif
+#ifndef LLAMA_DEFAULT_RMS_EPS
+#define LLAMA_DEFAULT_RMS_EPS 5e-6f
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -87,6 +91,7 @@ extern "C" {
int32_t n_ctx; // text context
int32_t n_batch; // prompt processing batch size
int32_t n_gqa; // grouped-query attention (TEMP - will be moved to model hparams)
+ float rms_norm_eps; // rms norm epsilon (TEMP - will be moved to model hparams)
int32_t n_gpu_layers; // number of layers to store in VRAM
int32_t main_gpu; // the GPU that is used for scratch and small tensors