Skip to content

Commit

Permalink
修改COPY-FROM No.14 incubate (#6003)
Browse files Browse the repository at this point in the history 
Signed-off-by: jjyaoao <jjyaoao@126.com>
  • Loading branch information
@jjyaoao
jjyaoao authored Jul 18, 2023
1 parent d1445d7 commit 0e2b51f
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 197 deletions.
93 changes: 3 additions & 90 deletions docs/api/paddle/incubate/LookAhead_cn.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,68 +25,7 @@ Lookahead 每 k 次训练迭代更新 slow_params 和 fast_params，如下所示
代码示例
:::::::::

.. code-block:: python
import numpy as np
import paddle
import paddle.nn as nn
BATCH_SIZE = 16
BATCH_NUM = 4
EPOCH_NUM = 4
IMAGE_SIZE = 784
CLASS_NUM = 10
# define a random dataset
class RandomDataset(paddle.io.Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples
def __getitem__(self, idx):
image = np.random.random([IMAGE_SIZE]).astype('float32')
label = np.random.randint(0, CLASS_NUM - 1,
(1, )).astype('int64')
return image, label
def __len__(self):
return self.num_samples
class LinearNet(nn.Layer):
def __init__(self):
super().__init__()
self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
self.bias = self._linear.bias
@paddle.jit.to_static
def forward(self, x):
return self._linear(x)
def train(layer, loader, loss_fn, opt):
for epoch_id in range(EPOCH_NUM):
for batch_id, (image, label) in enumerate(loader()):
out = layer(image)
loss = loss_fn(out, label)
loss.backward()
opt.step()
opt.clear_grad()
print("Train Epoch {} batch {}: loss = {}".format(
epoch_id, batch_id, np.mean(loss.numpy())))
layer = LinearNet()
loss_fn = nn.CrossEntropyLoss()
optimizer = paddle.optimizer.SGD(learning_rate=0.1, parameters=layer.parameters())
lookahead = paddle.incubate.LookAhead(optimizer, alpha=0.2, k=5)
# create data loader
dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
loader = paddle.io.DataLoader(
dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=2)
train(layer, loader, loss_fn, lookahead)
COPY-FROM: paddle.incubate.LookAhead

方法
:::::::::
Expand All @@ -104,20 +43,7 @@ None。

**代码示例**

.. code-block:: python
import paddle
import numpy as np
inp = paddle.to_tensor(np.random.random([1, 10]).astype('float32'))
linear = paddle.nn.Linear(10, 1)
out = linear(inp)
loss = paddle.mean(out)
sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
lookahead = paddle.incubate.LookAhead(sgd, alpha=0.2, k=5)
loss.backward()
lookahead.step()
lookahead.clear_grad()
COPY-FROM: paddle.incubate.LookAhead.step

minimize(loss, startup_program=None, parameters=None, no_grad_set=None)
'''''''''
Expand All @@ -137,17 +63,4 @@ tuple: tuple (optimize_ops, params_grads)，由 ``minimize`` 添加的操作列

**代码示例**

.. code-block:: python
import paddle
import numpy as np
inp = paddle.to_tensor(np.random.random([1, 10]).astype('float32'))
linear = paddle.nn.Linear(10, 1)
out = linear(inp)
loss = paddle.mean(out)
sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
lookahead = paddle.incubate.LookAhead(sgd, alpha=0.2, k=5)
loss.backward()
lookahead.minimize(loss)
lookahead.clear_grad()
COPY-FROM: paddle.incubate.LookAhead.minimize
56 changes: 2 additions & 54 deletions docs/api/paddle/incubate/optimizer/functional/minimize_bfgs_cn.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,61 +64,9 @@ BFGS 具体原理参考书籍 Jorge Nocedal, Stephen J. Wright, Numerical Optimi
代码示例 1
::::::::::

.. code-block:: python
import paddle
# 随机模拟一批输入数据
inputs = paddle.normal(shape=(100, 1))
labels = inputs * 2.0
# 定义 loss 函数
def loss(w):
y = w * inputs
return paddle.nn.functional.square_error_cost(y, labels).mean()
# 初始化权重参数
w = paddle.normal(shape=(1,))
# 调用 bfgs 方法求解使得 loss 最小的权重，并更新参数
for epoch in range(0, 10):
# 调用 bfgs 方法优化 loss，注意返回的第三个参数表示权重
w_update= paddle.incubate.optimizer.functional.minimize_bfgs(loss, w)[2]
# 使用 paddle.assign，以 inplace 方式更新参数
paddle.assign(w_update, w)
COPY-FROM: paddle.incubate.optimizer.functional.minimize_bfgs:code-example1

代码示例 2
::::::::::

.. code-block:: python
import paddle
def flatten(x):
return x.flatten()
def unflatten(x):
return x.reshape((2,2))
# 假设网络参数超过一维
def net(x):
assert len(x.shape) > 1
return x.square().mean()
# 待优化函数
def bfgs_f(flatten_x):
return net(unflatten(flatten_x))
x = paddle.rand([2,2])
for i in range(0, 10):
# 使用 minimize_bfgs 前，先将 x 展平
x_update = paddle.incubate.optimizer.functional.minimize_bfgs(bfgs_f, flatten(x))[2]
# 将 x_update unflatten，然后更新参数
paddle.assign(unflatten(x_update), x)
COPY-FROM: paddle.incubate.optimizer.functional.minimize_bfgs:code-example2
55 changes: 2 additions & 53 deletions docs/api/paddle/incubate/optimizer/functional/minimize_lbfgs_cn.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,61 +58,10 @@ LBFGS 具体原理参考书籍 Jorge Nocedal, Stephen J. Wright, Numerical Optim
代码示例 1：
::::::::::

.. code-block:: python
import paddle
# 随机模拟一批输入数据
inputs = paddle.normal(shape=(100, 1))
labels = inputs * 2.0
# 定义 loss 函数
def loss(w):
y = w * inputs
return paddle.nn.functional.square_error_cost(y, labels).mean()
# 初始化权重参数
w = paddle.normal(shape=(1,))
# 调用 bfgs 方法求解使得 loss 最小的权重，并更新参数
for epoch in range(0, 10):
# 调用 bfgs 方法优化 loss，注意返回的第三个参数表示权重
w_update= paddle.incubate.optimizer.functional.minimize_lbfgs(loss, w)[2]
# 使用 paddle.assign，以 inplace 方式更新参数
paddle.assign(w_update, w)
COPY-FROM: paddle.incubate.optimizer.functional.minimize_lbfgs:code-example1


代码示例 2：输入参数维度超过一维
::::::::::

.. code-block:: python
import paddle
def flatten(x):
return x.flatten()
def unflatten(x):
return x.reshape((2,2))
# 假设网络参数超过一维
def net(x):
assert len(x.shape) > 1
return x.square().mean()
# 待优化函数
def bfgs_f(flatten_x):
return net(unflatten(flatten_x))
x = paddle.rand([2,2])
for i in range(0, 10):
# 使用 minimize_lbfgs 前，先将 x 展平
x_update = paddle.incubate.optimizer.functional.minimize_lbfgs(bfgs_f, flatten(x))[2]
# 将 x_update unflatten，然后更新参数
paddle.assign(unflatten(x_update), x)
COPY-FROM: paddle.incubate.optimizer.functional.minimize_lbfgs:code-example2

0 comments on commit 0e2b51f

Please sign in to comment.