novarobot
/
llama.cpp
mirror of https://github.com/novarobot/llama.cpp
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove direct access to std streams from "run"

Browse Source 
The goal is to allow running "run" while connected to other streams,
such as TCP sockets.

Signed-off-by: Thiago Padilha <thiago@padilha.cc>
tcp_server
Thiago Padilha 3 years ago
parent b7f1fa6d8c
commit bf44faa0ee
No known key found for this signature in database
GPG Key ID: 309C78E5ED1B3D5E
3 changed files with 40 additions and 30 deletions
Show Stats Download Patch File Download Diff File

4
main.cpp
Unescape Escape View File

@ -1,6 +1,8 @@
#include "run.h" #include "run.h"
#include "ggml.h" #include "ggml.h"
#include <iostream>
std::vector<double> softmax(const std::vector<float>& logits) { std::vector<double> softmax(const std::vector<float>& logits) {
std::vector<double> probs(logits.size()); std::vector<double> probs(logits.size());
@ -123,5 +125,5 @@ int main(int argc, char ** argv) {
exit(0); exit(0);
} }
return run(ctx, params); return run(ctx, params, std::cin, stdout, stderr);
} }

60
run.cpp
Unescape Escape View File

@ -44,7 +44,7 @@ enum console_state {
static console_state con_st = CONSOLE_STATE_DEFAULT; static console_state con_st = CONSOLE_STATE_DEFAULT;
static bool con_use_color = false; static bool con_use_color = false;
void set_console_state(console_state new_st) void set_console_state(FILE *stream, console_state new_st)
{ {
if (!con_use_color) return; if (!con_use_color) return;
// only emit color code if state changed // only emit color code if state changed
@ -52,13 +52,13 @@ void set_console_state(console_state new_st)
con_st = new_st; con_st = new_st;
switch(con_st) { switch(con_st) {
case CONSOLE_STATE_DEFAULT: case CONSOLE_STATE_DEFAULT:
printf(ANSI_COLOR_RESET); fprintf(stream, ANSI_COLOR_RESET);
return; return;
case CONSOLE_STATE_PROMPT: case CONSOLE_STATE_PROMPT:
printf(ANSI_COLOR_YELLOW); fprintf(stream, ANSI_COLOR_YELLOW);
return; return;
case CONSOLE_STATE_USER_INPUT: case CONSOLE_STATE_USER_INPUT:
printf(ANSI_BOLD ANSI_COLOR_GREEN); fprintf(stream, ANSI_BOLD ANSI_COLOR_GREEN);
return; return;
} }
} }
@ -68,7 +68,7 @@ static bool is_interacting = false;
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
void sigint_handler(int signo) { void sigint_handler(int signo) {
set_console_state(CONSOLE_STATE_DEFAULT); set_console_state(stdout, CONSOLE_STATE_DEFAULT);
printf("\n"); // this also force flush stdout. printf("\n"); // this also force flush stdout.
if (signo == SIGINT) { if (signo == SIGINT) {
if (!is_interacting) { if (!is_interacting) {
@ -80,13 +80,17 @@ void sigint_handler(int signo) {
} }
#endif #endif
int run(llama_context * ctx, gpt_params params) { int run(llama_context * ctx,
gpt_params params,
std::istream & instream,
FILE *outstream,
FILE *errstream) {
if (params.seed <= 0) { if (params.seed <= 0) {
params.seed = time(NULL); params.seed = time(NULL);
} }
fprintf(stderr, "%s: seed = %d\n", __func__, params.seed); fprintf(errstream, "%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed); std::mt19937 rng(params.seed);
if (params.random_prompt) { if (params.random_prompt) {
@ -138,13 +142,13 @@ int run(llama_context * ctx, gpt_params params) {
params.interactive = true; params.interactive = true;
} }
fprintf(stderr, "\n"); fprintf(errstream, "\n");
fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); fprintf(errstream, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); fprintf(errstream, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
for (int i = 0; i < (int) embd_inp.size(); i++) { for (int i = 0; i < (int) embd_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], llama_token_to_str(ctx, embd_inp[i])); fprintf(errstream, "%6d -> '%s'\n", embd_inp[i], llama_token_to_str(ctx, embd_inp[i]));
} }
fprintf(stderr, "\n"); fprintf(errstream, "\n");
if (params.interactive) { if (params.interactive) {
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
struct sigaction sigint_action; struct sigaction sigint_action;
@ -156,16 +160,16 @@ int run(llama_context * ctx, gpt_params params) {
signal(SIGINT, sigint_handler); signal(SIGINT, sigint_handler);
#endif #endif
fprintf(stderr, "%s: interactive mode on.\n", __func__); fprintf(errstream, "%s: interactive mode on.\n", __func__);
if(params.antiprompt.size()) { if(params.antiprompt.size()) {
for (auto antiprompt : params.antiprompt) { for (auto antiprompt : params.antiprompt) {
fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str()); fprintf(errstream, "Reverse prompt: '%s'\n", antiprompt.c_str());
} }
} }
} }
fprintf(stderr, "sampling parameters: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty); fprintf(errstream, "sampling parameters: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty);
fprintf(stderr, "\n\n"); fprintf(errstream, "\n\n");
std::vector<llama_token> embd; std::vector<llama_token> embd;
@ -174,7 +178,7 @@ int run(llama_context * ctx, gpt_params params) {
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0); std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
if (params.interactive) { if (params.interactive) {
fprintf(stderr, "== Running in interactive mode. ==\n" fprintf(errstream, "== Running in interactive mode. ==\n"
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
" - Press Ctrl+C to interject at any time.\n" " - Press Ctrl+C to interject at any time.\n"
#endif #endif
@ -199,13 +203,13 @@ int run(llama_context * ctx, gpt_params params) {
} }
#endif #endif
// the first thing we will do is to output the prompt, so set color accordingly // the first thing we will do is to output the prompt, so set color accordingly
set_console_state(CONSOLE_STATE_PROMPT); set_console_state(outstream, CONSOLE_STATE_PROMPT);
while (remaining_tokens > 0 || params.interactive) { while (remaining_tokens > 0 || params.interactive) {
// predict // predict
if (embd.size() > 0) { if (embd.size() > 0) {
if (llama_eval(ctx, embd.data(), embd.size(), n_past, params.n_threads)) { if (llama_eval(ctx, embd.data(), embd.size(), n_past, params.n_threads)) {
fprintf(stderr, "%s : failed to eval\n", __func__); fprintf(errstream, "%s : failed to eval\n", __func__);
return 1; return 1;
} }
} }
@ -263,13 +267,13 @@ int run(llama_context * ctx, gpt_params params) {
// display text // display text
if (!input_noecho) { if (!input_noecho) {
for (auto id : embd) { for (auto id : embd) {
printf("%s", llama_token_to_str(ctx, id)); fprintf(outstream, "%s", llama_token_to_str(ctx, id));
} }
fflush(stdout); fflush(outstream);
} }
// reset color to default if we there is no pending user input // reset color to default if we there is no pending user input
if (!input_noecho && (int)embd_inp.size() == input_consumed) { if (!input_noecho && (int)embd_inp.size() == input_consumed) {
set_console_state(CONSOLE_STATE_DEFAULT); set_console_state(outstream, CONSOLE_STATE_DEFAULT);
} }
// in interactive mode, and not currently processing queued inputs; // in interactive mode, and not currently processing queued inputs;
@ -290,20 +294,20 @@ int run(llama_context * ctx, gpt_params params) {
} }
if (is_interacting) { if (is_interacting) {
// potentially set color to indicate we are taking user input // potentially set color to indicate we are taking user input
set_console_state(CONSOLE_STATE_USER_INPUT); set_console_state(outstream, CONSOLE_STATE_USER_INPUT);
if (params.instruct) { if (params.instruct) {
input_consumed = embd_inp.size(); input_consumed = embd_inp.size();
embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end()); embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());
printf("\n> "); fprintf(outstream, "\n> ");
} }
std::string buffer; std::string buffer;
std::string line; std::string line;
bool another_line = true; bool another_line = true;
do { do {
std::getline(std::cin, line); std::getline(instream, line);
if (line.empty() || line.back() != '\\') { if (line.empty() || line.back() != '\\') {
another_line = false; another_line = false;
} else { } else {
@ -313,7 +317,7 @@ int run(llama_context * ctx, gpt_params params) {
} while (another_line); } while (another_line);
// done taking input, reset color // done taking input, reset color
set_console_state(CONSOLE_STATE_DEFAULT); set_console_state(outstream, CONSOLE_STATE_DEFAULT);
auto line_inp = ::llama_tokenize(ctx, buffer, false); auto line_inp = ::llama_tokenize(ctx, buffer, false);
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end()); embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
@ -334,7 +338,7 @@ int run(llama_context * ctx, gpt_params params) {
if (params.interactive) { if (params.interactive) {
is_interacting = true; is_interacting = true;
} else { } else {
fprintf(stderr, " [end of text]\n"); fprintf(errstream, " [end of text]\n");
break; break;
} }
} }
@ -354,7 +358,7 @@ int run(llama_context * ctx, gpt_params params) {
llama_free(ctx); llama_free(ctx);
set_console_state(CONSOLE_STATE_DEFAULT); set_console_state(outstream, CONSOLE_STATE_DEFAULT);
return 0; return 0;
} }

6
run.h
Unescape Escape View File

@ -3,4 +3,8 @@
#include "llama.h" #include "llama.h"
#include "utils.h" #include "utils.h"
int run(llama_context * ctx, gpt_params params); int run(llama_context * ctx,
gpt_params params,
std::istream & instream,
FILE *outstream,
FILE *errstream);

Write Preview
Loading…
Cancel
Save