-
Notifications
You must be signed in to change notification settings - Fork 5.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add ctr table depends * code style * fix * fix * fix naming * rename * rename
- Loading branch information
1 parent
7253398
commit d64f7b3
Showing
10 changed files
with
937 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #pragma once | ||
| #include <assert.h> | ||
| #include <time.h> | ||
| #include <atomic> | ||
| #include <random> | ||
|
|
||
| namespace paddle { | ||
| namespace distributed { | ||
|
|
||
| // Get time in seconds. | ||
| inline double current_realtime() { | ||
| struct timespec tp; | ||
| clock_gettime(CLOCK_REALTIME, &tp); | ||
| return tp.tv_sec + tp.tv_nsec * 1e-9; | ||
| } | ||
|
|
||
| inline std::default_random_engine& local_random_engine() { | ||
| struct engine_wrapper_t { | ||
| std::default_random_engine engine; | ||
| engine_wrapper_t() { | ||
| static std::atomic<unsigned long> x(0); // NOLINT | ||
| std::seed_seq sseq = { | ||
| x++, x++, x++, (unsigned long)(current_realtime() * 1000)}; // NOLINT | ||
| engine.seed(sseq); | ||
| } | ||
| }; | ||
| thread_local engine_wrapper_t r; | ||
| return r.engine; | ||
| } | ||
|
|
||
| template <class T = double> | ||
| std::uniform_real_distribution<T>& local_uniform_real_distribution() { | ||
| thread_local std::uniform_real_distribution<T> distr; | ||
| assert(distr.a() == 0.0 && distr.b() == 1.0); | ||
| return distr; | ||
| } | ||
|
|
||
| template <class T = double> | ||
| T uniform_real() { | ||
| return local_uniform_real_distribution<T>()(local_random_engine()); | ||
| } | ||
|
|
||
| template <class T = double> | ||
| T uniform_real(T a, T b) { | ||
| if (a == b) { | ||
| return a; | ||
| } | ||
| return (T)(a + uniform_real<T>() * (b - a)); | ||
| } | ||
| } // namespace distributed | ||
| } // namespace paddle |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| // Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <ThreadPool.h> | ||
| #include <functional> | ||
| #include <future> // NOLINT | ||
| #include <memory> | ||
| #include <string> | ||
| #include <thread> // NOLINT | ||
| #include <unordered_map> | ||
| #include <unordered_set> | ||
| #include <utility> | ||
| #include <vector> | ||
| #include "gflags/gflags.h" | ||
|
|
||
| #include "butil/object_pool.h" | ||
| #include "paddle/fluid/distributed/common/utils.h" | ||
| #include "paddle/fluid/distributed/table/depends/initializers.h" | ||
| #include "paddle/fluid/distributed/thirdparty/round_robin.h" | ||
| #include "paddle/fluid/framework/generator.h" | ||
| #include "paddle/fluid/framework/lod_tensor.h" | ||
| #include "paddle/fluid/framework/rw_lock.h" | ||
| #include "paddle/fluid/framework/selected_rows.h" | ||
| #include "paddle/fluid/framework/tensor.h" | ||
| #include "paddle/fluid/framework/threadpool.h" | ||
| #include "paddle/fluid/framework/variable.h" | ||
| #include "paddle/fluid/platform/device_context.h" | ||
| #include "paddle/fluid/platform/enforce.h" | ||
| #include "paddle/fluid/platform/place.h" | ||
| #include "paddle/fluid/platform/port.h" | ||
| #include "paddle/fluid/string/printf.h" | ||
| #include "paddle/fluid/string/string_helper.h" | ||
|
|
||
| namespace paddle { | ||
| namespace distributed { | ||
|
|
||
| static const int CTR_SPARSE_SHARD_BUCKET_NUM_BITS = 6; | ||
| static const size_t CTR_SPARSE_SHARD_BUCKET_NUM = | ||
| static_cast<size_t>(1) << CTR_SPARSE_SHARD_BUCKET_NUM_BITS; | ||
|
|
||
| class FixedFeatureValue { | ||
| public: | ||
| FixedFeatureValue() {} | ||
| ~FixedFeatureValue() {} | ||
| float *data() { return data_.data(); } | ||
| size_t size() { return data_.size(); } | ||
| void resize(size_t size) { data_.resize(size); } | ||
| void shrink_to_fit() { data_.shrink_to_fit(); } | ||
|
|
||
| private: | ||
| std::vector<float> data_; | ||
| }; | ||
|
|
||
| class SparseTableShard { | ||
| public: | ||
| typedef typename robin_hood::unordered_map<uint64_t, FixedFeatureValue *> | ||
| map_type; | ||
| SparseTableShard() {} | ||
| ~SparseTableShard() {} | ||
|
|
||
| FixedFeatureValue *Init(const uint64_t &id) { | ||
| size_t hash = hasher_(id); | ||
| size_t bucket = compute_bucket(hash); | ||
| auto &table = values_[bucket]; | ||
|
|
||
| FixedFeatureValue *value = nullptr; | ||
| value = butil::get_object<FixedFeatureValue>(); | ||
| table[id] = value; | ||
| return value; | ||
| } | ||
|
|
||
| // dont judge if (has(id)) | ||
| float *Get(const uint64_t &id) { | ||
| size_t hash = hasher_(id); | ||
| size_t bucket = compute_bucket(hash); | ||
| auto &table = values_[bucket]; | ||
|
|
||
| // auto &value = table.at(id); | ||
| // return value->data_.data(); | ||
| auto res = table.find(id); | ||
| FixedFeatureValue *value = res->second; | ||
| return value->data(); | ||
| } | ||
|
|
||
| // for load, to reset count, unseen_days | ||
| FixedFeatureValue *GetValue(const uint64_t &id) { | ||
| size_t hash = hasher_(id); | ||
| size_t bucket = compute_bucket(hash); | ||
|
|
||
| auto &table = values_[bucket]; | ||
| auto res = table.find(id); | ||
| return res->second; | ||
| } | ||
|
|
||
| void erase(uint64_t feasign) { | ||
| size_t hash = hasher_(feasign); | ||
| size_t bucket = compute_bucket(hash); | ||
| auto &table = values_[bucket]; | ||
|
|
||
| auto iter = table.find(feasign); | ||
| if (iter != table.end()) { | ||
| butil::return_object(iter->second); | ||
| iter = table.erase(iter); | ||
| } | ||
| } | ||
|
|
||
| void clear() {} | ||
|
|
||
| size_t compute_bucket(size_t hash) { | ||
| if (CTR_SPARSE_SHARD_BUCKET_NUM == 1) { | ||
| return 0; | ||
| } else { | ||
| return hash >> (sizeof(size_t) * 8 - CTR_SPARSE_SHARD_BUCKET_NUM_BITS); | ||
| } | ||
| } | ||
|
|
||
| map_type::iterator end() { | ||
| return values_[CTR_SPARSE_SHARD_BUCKET_NUM - 1].end(); | ||
| } | ||
|
|
||
| map_type::iterator Find(uint64_t id) { | ||
| size_t hash = hasher_(id); | ||
| size_t bucket = compute_bucket(hash); | ||
| auto &table = values_[bucket]; | ||
|
|
||
| auto got = table.find(id); | ||
| if (got == table.end()) { | ||
| return end(); | ||
| } else { | ||
| return got; | ||
| } | ||
| } | ||
|
|
||
| private: | ||
| bool Has(const uint64_t id) { | ||
| size_t hash = hasher_(id); | ||
| size_t bucket = compute_bucket(hash); | ||
| auto &table = values_[bucket]; | ||
|
|
||
| auto got = table.find(id); | ||
| if (got == table.end()) { | ||
| return false; | ||
| } else { | ||
| return true; | ||
| } | ||
| } | ||
|
|
||
| public: | ||
| map_type values_[CTR_SPARSE_SHARD_BUCKET_NUM]; | ||
| std::hash<uint64_t> hasher_; | ||
| }; | ||
|
|
||
| } // namespace distributed | ||
| } // namespace paddle |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.