Skip to content

Commit

Permalink
streaming working. Need to polish
Browse files Browse the repository at this point in the history
  • Loading branch information
@tybalex
tybalex committed Jul 10, 2024
1 parent 89971d6 commit 5829dfc
Show file tree
Hide file tree
Showing 2 changed files with 145 additions and 32 deletions.
89 changes: 77 additions & 12 deletions examples/server/server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3092,27 +3092,92 @@ int main(int argc, char ** argv) {
ctx_server.queue_results.remove_waiting_task_id(id_task);
} else {
const auto chunked_content_provider = [id_task, &ctx_server, completion_id](size_t, httplib::DataSink & sink) {
std::string leading_str = "";
bool is_function_call = false;
bool checked_function_call = false;
json last_result_data;
while (true) {
server_task_result result = ctx_server.queue_results.recv(id_task);
if (!result.error) {
std::vector<json> result_array = format_partial_response_oaicompat(result.data, completion_id);

for (auto it = result_array.begin(); it != result_array.end(); ++it) {
if (!it->empty()) {
const std::string str =
"data: " +
it->dump(-1, ' ', false, json::error_handler_t::replace) +
"\n\n";
LOG_VERBOSE("data stream", {{"to_send", str}});
if (!sink.write(str.c_str(), str.size())) {
ctx_server.queue_results.remove_waiting_task_id(id_task);
return false;
if (!checked_function_call) {
leading_str += result.data.value("content", "");
if (leading_str.length() >= 15) {
if (leading_str.find("starttoolcall") != std::string::npos) {
is_function_call = true;
printf("####set function call to TRUE because of str: %s", leading_str.c_str());
}
checked_function_call = true;
}
}
else {
if (is_function_call) {
leading_str += result.data.value("content", "");
}
else {
if (leading_str.length() > 0) {
result.data["content"] = leading_str + result.data.value("content", "");
leading_str = "";
}
std::vector<json> result_array = format_partial_response_oaicompat(result.data, completion_id);

for (auto it = result_array.begin(); it != result_array.end(); ++it) {
if (!it->empty()) {
const std::string str =
"data: " +
it->dump(-1, ' ', false, json::error_handler_t::replace) +
"\n\n";
LOG_VERBOSE("data stream", {{"to_send", str}});
if (!sink.write(str.c_str(), str.size())) {
ctx_server.queue_results.remove_waiting_task_id(id_task);
return false;
}
}
}
}
}

if (result.stop) {
if (leading_str.length() > 0) {
printf("### Entering last stage?\nwith leading str as : %s \n\n", leading_str.c_str());
last_result_data["content"] = leading_str;
std::vector<json> result_array = format_partial_response_oaicompat(last_result_data, completion_id);

for (auto it = result_array.begin(); it != result_array.end(); ++it) {
if (!it->empty()) {
const std::string str =
"data: " +
it->dump(-1, ' ', false, json::error_handler_t::replace) +
"\n\n";
LOG_VERBOSE("data stream", {{"to_send", str}});
if (!sink.write(str.c_str(), str.size())) {
ctx_server.queue_results.remove_waiting_task_id(id_task);
return false;
}
}
}

result_array = format_partial_response_oaicompat(result.data, completion_id);

for (auto it = result_array.begin(); it != result_array.end(); ++it) {
if (!it->empty()) {
const std::string str =
"data: " +
it->dump(-1, ' ', false, json::error_handler_t::replace) +
"\n\n";
LOG_VERBOSE("data stream", {{"to_send", str}});
if (!sink.write(str.c_str(), str.size())) {
ctx_server.queue_results.remove_waiting_task_id(id_task);
return false;
}
}
}
}

break;
}
else {
last_result_data = result.data;
}
} else {
const std::string str =
"error: " +
Expand Down
88 changes: 68 additions & 20 deletions examples/server/utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,7 @@ static std::vector<json> format_partial_response_oaicompat(json result, const st
bool stopped_eos = json_value(result, "stopped_eos", false);
bool stopped_limit = json_value(result, "stopped_limit", false);
std::string content = json_value(result, "content", std::string(""));
std::vector<json> parsed_content = rubra_fc_json_tool_extractor(content);

std::string finish_reason;
if (stopped_word || stopped_eos) {
Expand Down Expand Up @@ -562,18 +563,41 @@ static std::vector<json> format_partial_response_oaicompat(json result, const st
{"model", modelname},
{"object", "chat.completion.chunk"}};

json second_ret = json{
{"choices", json::array({json{{"finish_reason", nullptr},
{"index", 0},
{"delta", json{
{"content", content}}}
}})},
{"created", t},
{"id", completion_id},
{"model", modelname},
{"object", "chat.completion.chunk"}};

return std::vector<json>({initial_ret, second_ret});
if (parsed_content.empty()) {
json second_ret = json{
{"choices", json::array({json{{"finish_reason", nullptr},
{"index", 0},
{"delta", json{
{"content", content}}}
}})},
{"created", t},
{"id", completion_id},
{"model", modelname},
{"object", "chat.completion.chunk"}};

return std::vector<json>({initial_ret, second_ret});
}
else {
std::vector<json> oai_format_tool_calls;
for (size_t i = 0; i < parsed_content.size(); ++i) {
const auto &pc = parsed_content[i];
// Use 'pc' and 'i' as needed
json tool_call;
tool_call["id"] = pc["id"];
tool_call["type"] = "function";

tool_call["function"] = json{
{"name" , pc["name"]},
{"arguments" , pc["kwargs"]},
};
oai_format_tool_calls.push_back(tool_call);
}
choices = json::array({json{{"finish_reason", nullptr},
{"index", 0},
{"delta", json{{"tool_calls", oai_format_tool_calls},
{"role", "assistant"}}}}});
}

}
} else {
// Some idiosyncrasy in task processing logic makes several trailing calls
Expand All @@ -582,14 +606,38 @@ static std::vector<json> format_partial_response_oaicompat(json result, const st
return std::vector<json>({json::object()});
}

choices = json::array({json{
{"finish_reason", nullptr},
{"index", 0},
{"delta",
json{
{"content", content},
}},
}});
if (parsed_content.empty()) {
choices = json::array({json{
{"finish_reason", nullptr},
{"index", 0},
{"delta",
json{
{"content", content},
}},
}});
}
else {
std::vector<json> oai_format_tool_calls;
for (size_t i = 0; i < parsed_content.size(); ++i) {
const auto &pc = parsed_content[i];
// Use 'pc' and 'i' as needed
json tool_call;
tool_call["id"] = pc["id"];
tool_call["type"] = "function";
tool_call["index"] = i;

tool_call["function"] = json{
{"name" , pc["name"]},
{"arguments" , pc["kwargs"]},
};
oai_format_tool_calls.push_back(tool_call);
}
choices = json::array({json{{"finish_reason", nullptr},
{"index", 0},
{"delta", json{{"tool_calls", oai_format_tool_calls},
{"role", "assistant"}}}}});
}

}
}

Expand Down

0 comments on commit 5829dfc

Please sign in to comment.