Skip to content

Commit 7b7b7db

Browse files
mlimli
committed
moving codes
1 parent c478e73 commit 7b7b7db

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

115 files changed

+14117
-2
lines changed

‎README.org‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,25 @@ use, high performance supercomputer clusters.
3434
replication architecture that efficiently stores data on multiple server nodes
3535
to enable fast (in less than 1 second) recovery from node failure.
3636

37-
** Architecture Overview
37+
** Architecture
3838

3939
The parameter server architecture has two classes of
4040
nodes: Each *server* node maintains a partition of the globally
4141
shared parameters. They communicate with each other to replicate and/or to
42-
migrate parameters for reliability and scaling. The =client=
42+
migrate parameters for reliability and scaling. The *client*
4343
nodes perform the bulk of the computation. Each client
4444
typically stores locally a portion of the training data, computing
4545
local statistics such as gradients. Clients communicate only with the
4646
server nodes, updating and retrieving the shared parameters. Clients
4747
may be added or removed.
4848

4949
[[./doc/img/arch2.png]]
50+
51+
** APIs
52+
53+
54+
** Sample Codes
55+
56+
** Current Progress
57+
58+
**

‎doc/img/arch2.png‎

39.4 KB
Loading

‎src/Makefile‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
CC = g++
2+
3+
# OPT = -O3
4+
OPT = -O0 -ggdb
5+
6+
#-Wconversion -fPIC
7+
WARN = -Wall -Wno-unused-function -finline-functions
8+
INCPATH = -I. -I/usr/local/include \
9+
-I../third_party/gtest-1.7.0/include \
10+
-I../third_party
11+
CFLAGS = -std=c++0x $(WARN) $(OPT) $(INCPATH)
12+
13+
LDFLAGS = -L/usr/local/lib libps.a -lgflags -lpthread -lzmq -lprotobuf -lglog
14+
#GTEST = ../third_party/gtest-1.7.0/libgtest_main.a \
15+
# ../third_party/gtest-1.7.0/libgtest.a
16+
GTEST = -L../third_party/gtest-1.7.0 -lgtest_main -lgtest
17+
#GTEST = ../third_party/gtest/libgtest_main.a
18+
19+
OBJECTS = \
20+
./proto/nodemgt.pb.o \
21+
./proto/header.pb.o \
22+
./util/crc32c.o \
23+
./util/key.o \
24+
./util/mail.o \
25+
./util/join.o \
26+
./util/stringpiece.o \
27+
./util/stringprintf.o \
28+
./util/futurepool.o \
29+
./util/md5.o \
30+
./util/hashfunc.o \
31+
./system/van.o \
32+
./system/node.o \
33+
./system/node_group.o \
34+
./system/workload.o \
35+
./system/postmaster.o \
36+
./system/postoffice.o \
37+
./system/hashring.o \
38+
./system/dht.o \
39+
./system/replica_manager.o \
40+
./system/aggregator.o \
41+
./box/consistency.o \
42+
./box/item.o \
43+
./box/container.o \
44+
./box/sparse_vector.o \
45+
./algo/inference.o \
46+
./algo/graddesc.o
47+
#./box/vectors.o
48+
49+
DEPS = \
50+
util/rawarray.h
51+
52+
53+
TESTS = \
54+
vectors_test \
55+
van_test \
56+
fault_tolerance_press \
57+
postmaster_test \
58+
# replica_test \
59+
key_test \
60+
xarray_test \
61+
eigen3_test \
62+
vector_test \
63+
item_test \
64+
densevector_test \
65+
sparse_vector_test \
66+
# item_test \
67+
consistency_test \
68+
workload_test \
69+
aggregator_test \
70+
common_test
71+
72+
# PWD := $(shell pwd)
73+
#r: clean all
74+
all: $(TESTS) ps gtest
75+
76+
clean:
77+
rm -f */*.o *.a ps
78+
79+
test: # $(TESTS)
80+
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
81+
82+
%.o: %.cc %.h
83+
$(CC) $(CFLAGS) -c $< -o $@
84+
85+
libps.a: $(OBJECTS) $(DEPS) gtest
86+
ar crv libps.a $(OBJECTS)
87+
88+
./proto/%.pb.cc ./proto/%.pb.h : ./proto/%.proto
89+
protoc --cpp_out=. $<
90+
91+
#base:
92+
# make -C $(PWD)/../third_party/ #M=$(PWD)
93+
94+
gtest:
95+
cmake ../third_party/gtest-1.7.0/CMakeLists.txt
96+
make -C ../third_party/gtest-1.7.0/
97+
98+
ps: libps.a ps.cc
99+
$(CC) $(CFLAGS) ps.cc $(LDFLAGS) -o parse
100+
101+
fault_tolerance_press: algo/fault_tolerance_press.cc libps.a algo/fault_tolerance_press.cc
102+
$(CC) $(CFLAGS) algo/fault_tolerance_press.cc $(LDFLAGS) -o fault_tolerance_press
103+
104+
common_test : util/common_test.cc
105+
$(CC) $(CFLAGS) util/common_test.cc $(GTEST) $(LDFLAGS) -o $@
106+
107+
hashfunc_test: util/hashfunc_test.cc
108+
$(CC) $(CFLAGS) util/hashfunc_test.cc $(GTEST) $(LDFLAGS) -o $@
109+
110+
hashring_test: system/hashring_test.cc
111+
$(CC) $(CFLAGS) system/hashring_test.cc $(GTEST) $(LDFLAGS) -o $@
112+
113+
metadata_test : box/metadata_test.cc
114+
$(CC) $(CFLAGS) box/metadata_test.cc $(GTEST) $(LDFLAGS) -o $@
115+
116+
workload_test : system/workload_test.cc libps.a
117+
$(CC) $(CFLAGS) system/workload_test.cc $(GTEST) $(LDFLAGS) -o $@
118+
119+
van_test : system/van_test.cc libps.a
120+
$(CC) $(CFLAGS) system/van_test.cc $(GTEST) $(LDFLAGS) -o $@
121+
122+
consistency_test : box/consistency_test.cc libps.a
123+
$(CC) $(CFLAGS) box/consistency_test.cc $(GTEST) $(LDFLAGS) -o $@
124+
125+
aggregator_test : box/aggregator_test.cc libps.a
126+
$(CC) $(CFLAGS) box/aggregator_test.cc $(GTEST) $(LDFLAGS) -o $@
127+
128+
item_test : box/item_test.cc libps.a
129+
$(CC) $(CFLAGS) box/item_test.cc $(GTEST) $(LDFLAGS) -o $@
130+
131+
key_test : util/key_test.cc libps.a
132+
$(CC) $(CFLAGS) util/key_test.cc $(GTEST) $(LDFLAGS) -o $@
133+
134+
xarray_test : util/xarray_test.cc util/xarray.h
135+
$(CC) $(CFLAGS) util/xarray_test.cc $(GTEST) $(LDFLAGS) -o $@
136+
137+
vectors_test : box/vectors_test.cc libps.a
138+
$(CC) $(CFLAGS) box/vectors_test.cc $(GTEST) $(LDFLAGS) -o $@
139+
140+
eigen3_test : util/eigen3_test.cc libps.a
141+
$(CC) $(CFLAGS) util/eigen3_test.cc $(GTEST) $(LDFLAGS) -o $@
142+
143+
sparsevector_test : libps.a
144+
$(CC) $(CFLAGS) box/sparsevector.cc $(GTEST) $(LDFLAGS) -o $@
145+
146+
sparse_vector_test : box/sparse_vector_test.cc libps.a
147+
$(CC) $(CFLAGS) box/sparse_vector_test.cc $(GTEST) $(LDFLAGS) -o $@
148+
149+
postmaster_test : system/postmaster_test.cc system/postmaster.cc system/postmaster.h libps.a
150+
$(CC) $(CFLAGS) system/postmaster_test.cc $(GTEST) $(LDFLAGS) -o $@
151+
152+
replica_test : system/replica_test.cc libps.a
153+
$(CC) $(CFLAGS) system/replica_test.cc $(GTEST) $(LDFLAGS) -o $@

‎src/README‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
-*- mode:org -*-
2+
format core:
3+
echo "core_%e_%p" | sudo tee /proc/sys/kernel/core_pattern
4+
* require packages:
5+
6+
they are under ../third_party/, just untar/configure/make/install
7+
8+
Gflag
9+
https://code.google.com/p/gflags/
10+
11+
Protobuf
12+
https://code.google.com/p/protobuf/
13+
14+
Zeromq 4.0.1
15+
http://zeromq.org/intro:get-the-software
16+
17+
* compiler
18+
19+
gcc >= 4.7
20+
21+
sudo apt-get install python-software-properties
22+
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
23+
sudo apt-get update
24+
sudo apt-get install gcc-4.7
25+
26+
clang >= 3.4
27+
http://llvm.org/apt/
28+
on ubuntu 12.10
29+
sudo add-apt-repository 'deb http://llvm.org/apt/quantal/ llvm-toolchain-quantal main'
30+
wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
31+
clang
32+
for ububutu, just sudo apt-get install clang
33+
you change CXX in Makefile into g++ if you prefer. we use clang mainly because it gives better
34+
compile error hints.
35+
36+
* Compile:
37+
38+
goto src/, then make

‎src/algo/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
all :
2+
make -C ..

‎src/algo/fault_tolerance_press.cc‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#include <stdlib.h>
2+
#include <vector>
3+
#include <time.h>
4+
#include <chrono>
5+
#include <gflags/gflags.h>
6+
#include "box/vectors.h"
7+
#include "util/xarray.h"
8+
#include "util/key.h"
9+
10+
namespace PS {
11+
12+
DECLARE_int32(my_rank);
13+
DECLARE_string(my_type);
14+
DECLARE_int32(num_client);
15+
DECLARE_int32(num_server);
16+
17+
DECLARE_int32(global_feature_num);
18+
DECLARE_int32(local_feature_num);
19+
20+
21+
void Server();
22+
void Client();
23+
24+
using std::vector;
25+
using std::string;
26+
27+
28+
vector<int> GenGlobalFeaId();
29+
30+
void Server() {
31+
vector<int> global_id = GenGlobalFeaId();
32+
XArray<Key> keys(FLAGS_global_feature_num);
33+
for (size_t i = 0; i < global_id.size(); ++i) {
34+
keys.set(i, global_id[i]);
35+
}
36+
Vectors<double> vec("ft", FLAGS_global_feature_num, 1);
37+
vec.SetAggregator(NodeGroup::kClients);
38+
while (1) {
39+
std::this_thread::sleep_for(seconds(1));
40+
LOG(WARNING) << "Vec[0]" << vec.Vec(0)[0];
41+
}
42+
}
43+
44+
vector<int> GenLocalFeaId();
45+
46+
void Client() {
47+
// generate local feature
48+
vector<int> local_id = GenLocalFeaId();
49+
50+
XArray<Key> keys(FLAGS_local_feature_num);
51+
for (size_t i = 0; i < local_id.size(); ++i) {
52+
keys.set(i, local_id[i]);
53+
}
54+
Vectors<double> vec("ft", FLAGS_global_feature_num, 1, keys);
55+
vec.SetMaxDelay(kint32max,kint32max);
56+
57+
while(1) {
58+
for (int i = 0; i < FLAGS_local_feature_num; ++i) {
59+
vec.Vec(0)[i] = vec.Vec(0)[i] + 1;
60+
}
61+
LOG(WARNING) << "push to server.";
62+
vec.PushPull(KeyRange::All(), {0}, kValue, {0}, kDelta);
63+
std::this_thread::sleep_for(std::chrono::milliseconds(10));
64+
}
65+
}
66+
67+
vector<int> GenLocalFeaId() {
68+
srand(time(NULL));
69+
CHECK(FLAGS_global_feature_num >= FLAGS_local_feature_num)
70+
<< "global fea num should be larger than local fea num";
71+
vector<int> feature_id;
72+
for (int i = 0; i < FLAGS_global_feature_num; ++i) {
73+
feature_id.push_back(i);
74+
}
75+
76+
std::random_shuffle(feature_id.begin(), feature_id.end());
77+
feature_id.resize(FLAGS_local_feature_num);
78+
79+
sort(feature_id.begin(), feature_id.end());
80+
return feature_id;
81+
}
82+
83+
vector<int> GenGlobalFeaId() {
84+
CHECK(FLAGS_global_feature_num >= FLAGS_local_feature_num)
85+
<< "global fea num should be larger than local fea num";
86+
vector<int> feature_id;
87+
for (int i = 0; i < FLAGS_global_feature_num; ++i) {
88+
feature_id.push_back(i);
89+
}
90+
return feature_id;
91+
}
92+
93+
}
94+
95+
int main(int argc, char *argv[]) {
96+
google::ParseCommandLineFlags(&argc, &argv, true);
97+
if (PS::FLAGS_my_type == "client") {
98+
PS::Client();
99+
} else {
100+
PS::Server();
101+
}
102+
return 0;
103+
}

0 commit comments

Comments
 (0)