Skip to content

Commit

Permalink
Merge pull request #85 from tiancaitzp/paddlebox
Browse files Browse the repository at this point in the history 
Paddlebox
  • Loading branch information
@tiancaitzp
tiancaitzp authored Jul 2, 2024
2 parents af3ee57 + 46f710e commit 9632d59
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 16 deletions.
2 changes: 1 addition & 1 deletion cmake/external/xpu.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ if (WITH_BOX_PS OR WITH_XPU_KP)
CACHE STRING "" FORCE)
#"https://klx-sdk-release-public.su.bcebos.com/xdnn/release/2.6.0.1/${XPU_XDNN_DIR_NAME}.tar.gz"
set(XPU_XDNN_URL
"https://klx-sdk-release-public.su.bcebos.com/xdnn_train/dev/paddlebox/20240617/${XPU_XDNN_DIR_NAME}.tar.gz"
"https://klx-sdk-release-public.su.bcebos.com/xdnn_train/dev/paddlebox/xdnn_master/20240618/${XPU_XDNN_DIR_NAME}.tar.gz"
CACHE STRING "" FORCE)
set(SCALOPUS_URL
"https://klx-sdk-release-public.su.bcebos.com/xdnn_train/dev/paddlebox/20230306/scalopus.tar.gz"
Expand Down
11 changes: 2 additions & 9 deletions paddle/fluid/framework/fleet/box_wrapper.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -903,7 +903,7 @@ class NanInfMetricMsg : public MetricMsg {
calculator = new BasicAucCalculator(mode_collect_in_gpu);
calculator->init(bucket_size);
}
virtual ~NanInfMetricMsg() { }
virtual ~NanInfMetricMsg() { }
void add_data(const Scope* exe_scope,
const paddle::platform::Place& place) override {
int label_len = 0;
Expand All @@ -921,7 +921,7 @@ class NanInfMetricMsg : public MetricMsg {
auto cal = GetCalculator();
auto pre_var_place = GetVarPlace(exe_scope, pred_varname_);
auto label_var_place = GetVarPlace(exe_scope, label_varname_);
cal->add_nan_inf_data(
cal->add_nan_inf_data(
pred_data, label_data, label_len, pre_var_place, label_var_place);
}
};
Expand Down Expand Up @@ -1160,13 +1160,6 @@ void BoxWrapper::PrintSyncTimer(int device, double train_span) {
<< ", dense nccl:" << dev.dense_nccl_timer.ElapsedSec()
<< ", sync stream:" << dev.dense_sync_timer.ElapsedSec()
<< ", wrapper xpu memory:" << dev.GpuMemUsed() << "MB";

auto env_str = std::getenv("ENABLE_FC_HIT_RATE");
if (env_str != nullptr && (strcmp(env_str, "true") == 0 || strcmp(env_str, "1") == 0)) {
auto place = platform::XPUPlace(device);
auto xpu_context = static_cast<platform::XPUDeviceContext*>(platform::DeviceContextPool::Instance().Get(place))->x_context();
LOG(INFO) << "xpu: " << device << ", fc_hit_rate: " << xpu_context->get_fc_hit_rate();
}
#else
LOG(WARNING) << "cpu: " << device << ", phase: " << phase_
<< ", train dnn: " << train_span
Expand Down
8 changes: 4 additions & 4 deletions paddle/fluid/operators/pull_box_extended_sparse_op.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ static void PullBoxExtendedSparseFunctor(
formatter.SetPrintTensorLod(true);
formatter.SetPrintTensorLayout(true);
// formatter.SetSummarize(static_cast<int64_t>(Attr<int>("summarize")));
// formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse_input.txt");
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".pull_sparse.txt");
std::string message = std::string("---embs_all_")+std::to_string(i)+std::string("---");
formatter.Print(*(outputs[i]), name, message);
}
Expand All @@ -203,7 +203,7 @@ static void PullBoxExtendedSparseFunctor(
formatter.SetPrintTensorLod(true);
formatter.SetPrintTensorLayout(true);
// formatter.SetSummarize(static_cast<int64_t>(Attr<int>("summarize")));
// formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse_input.txt");
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".pull_sparse.txt");
std::string message = std::string("---expand_all_")+std::to_string(i)+std::string("---");
formatter.Print(*(outputs_extend[i]), name, message);
}
Expand Down Expand Up @@ -303,7 +303,7 @@ static void PushBoxExtendedSparseFunctor(
formatter.SetPrintTensorLod(true);
formatter.SetPrintTensorLayout(true);
// formatter.SetSummarize(static_cast<int64_t>(Attr<int>("summarize")));
// formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse_input.txt");
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse.txt");
std::string message = std::string("---expand_all_")+std::to_string(i)+std::string("---");
formatter.Print(*(d_output_extend[i]), "print_" + name, message);
}
Expand All @@ -317,7 +317,7 @@ static void PushBoxExtendedSparseFunctor(
formatter.SetPrintTensorLod(true);
formatter.SetPrintTensorLayout(true);
// formatter.SetSummarize(static_cast<int64_t>(Attr<int>("summarize")));
// formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse_input.txt");
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse.txt");
std::string message = std::string("---embs_all_")+std::to_string(i)+std::string("---");
formatter.Print(*(d_output[i]), "print_" + name, message);
}
Expand Down
4 changes: 2 additions & 2 deletions paddle/fluid/operators/pull_box_sparse_op.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ static void PullBoxSparseFunctor(const framework::ExecutionContext &ctx) {
formatter.SetPrintTensorLod(true);
formatter.SetPrintTensorLayout(true);
// formatter.SetSummarize(static_cast<int64_t>(Attr<int>("summarize")));
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse_input.txt");
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".pull_sparse.txt");
std::string message = std::string("---embs_all_")+std::to_string(i)+std::string("---");
formatter.Print(*(outputs[i]), name, message);
}
Expand Down Expand Up @@ -245,7 +245,7 @@ static void PushBoxSparseFunctor(const framework::ExecutionContext &ctx) {
formatter.SetPrintTensorLod(true);
formatter.SetPrintTensorLayout(true);
// formatter.SetSummarize(static_cast<int64_t>(Attr<int>("summarize")));
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse_input.txt");
formatter.SetPrintFilePath("dev"+std::to_string(ctx.GetPlace().device)+".push_sparse.txt");
std::string message = std::string("---embs_all_")+std::to_string(i)+std::string("---");
formatter.Print(*(d_output[i]), name, message);
}
Expand Down

0 comments on commit 9632d59

Please sign in to comment.