直方图中的随机数

Question

假设我使用scipy / numpy创建直方图，所以我有两个数组：一个用于bin计数，一个用于bin边缘。如果我使用直方图来表示概率分布函数，我该如何从该分布中有效地生成随机数？

Answer 1

这可能是{Ophion的答案中np.random.choice所做的，但您可以构建归一化的累积密度函数，然后根据统一的随机数进行选择：

from __future__ import division
import numpy as np
import matplotlib.pyplot as plt

data = np.random.normal(size=1000)
hist, bins = np.histogram(data, bins=50)

bin_midpoints = bins[:-1] + np.diff(bins)/2
cdf = np.cumsum(hist)
cdf = cdf / cdf[-1]
values = np.random.rand(10000)
value_bins = np.searchsorted(cdf, values)
random_from_cdf = bin_midpoints[value_bins]

plt.subplot(121)
plt.hist(data, 50)
plt.subplot(122)
plt.hist(random_from_cdf, 50)
plt.show()

---

2D案例可以按如下方式完成：

data = np.column_stack((np.random.normal(scale=10, size=1000),
                        np.random.normal(scale=20, size=1000)))
x, y = data.T                        
hist, x_bins, y_bins = np.histogram2d(x, y, bins=(50, 50))
x_bin_midpoints = x_bins[:-1] + np.diff(x_bins)/2
y_bin_midpoints = y_bins[:-1] + np.diff(y_bins)/2
cdf = np.cumsum(hist.ravel())
cdf = cdf / cdf[-1]

values = np.random.rand(10000)
value_bins = np.searchsorted(cdf, values)
x_idx, y_idx = np.unravel_index(value_bins,
                                (len(x_bin_midpoints),
                                 len(y_bin_midpoints)))
random_from_cdf = np.column_stack((x_bin_midpoints[x_idx],
                                   y_bin_midpoints[y_idx]))
new_x, new_y = random_from_cdf.T

plt.subplot(121, aspect='equal')
plt.hist2d(x, y, bins=(50, 50))
plt.subplot(122, aspect='equal')
plt.hist2d(new_x, new_y, bins=(50, 50))
plt.show()

Answer 2

@Jaime解决方案很棒，但你应该考虑使用直方图的kde（核密度估计）。可以找到一个很好的解释为什么对直方图进行统计有问题，以及为什么应该使用kde代替here

我编辑了@ Jaime的代码，以展示如何使用scipy中的kde。它看起来几乎相同，但更好地捕获直方图生成器。

from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import gaussian_kde

def run():
    data = np.random.normal(size=1000)
    hist, bins = np.histogram(data, bins=50)

    x_grid = np.linspace(min(data), max(data), 1000)
    kdepdf = kde(data, x_grid, bandwidth=0.1)
    random_from_kde = generate_rand_from_pdf(kdepdf, x_grid)

    bin_midpoints = bins[:-1] + np.diff(bins) / 2
    random_from_cdf = generate_rand_from_pdf(hist, bin_midpoints)

    plt.subplot(121)
    plt.hist(data, 50, normed=True, alpha=0.5, label='hist')
    plt.plot(x_grid, kdepdf, color='r', alpha=0.5, lw=3, label='kde')
    plt.legend()
    plt.subplot(122)
    plt.hist(random_from_cdf, 50, alpha=0.5, label='from hist')
    plt.hist(random_from_kde, 50, alpha=0.5, label='from kde')
    plt.legend()
    plt.show()


def kde(x, x_grid, bandwidth=0.2, **kwargs):
    """Kernel Density Estimation with Scipy"""
    kde = gaussian_kde(x, bw_method=bandwidth / x.std(ddof=1), **kwargs)
    return kde.evaluate(x_grid)


def generate_rand_from_pdf(pdf, x_grid):
    cdf = np.cumsum(pdf)
    cdf = cdf / cdf[-1]
    values = np.random.rand(1000)
    value_bins = np.searchsorted(cdf, values)
    random_from_cdf = x_grid[value_bins]
    return random_from_cdf

Answer 3

也许是这样的。使用直方图的计数作为权重，并根据此权重选择指数值。

import numpy as np

initial=np.random.rand(1000)
values,indices=np.histogram(initial,bins=20)
values=values.astype(np.float32)
weights=values/np.sum(values)

#Below, 5 is the dimension of the returned array.
new_random=np.random.choice(indices[1:],5,p=weights)
print new_random

#[ 0.55141614  0.30226256  0.25243184  0.90023117  0.55141614]

Answer 4

我遇到与OP相同的问题，我想分享我对这个问题的处理方法。

关注Jaime answer和Noam Peled answer我使用Kernel Density Estimation (KDE)为2D问题构建了解决方案。

首先，让我们生成一些随机数据，然后从KDE计算它的Probability Density Function (PDF)。我将使用example available in SciPy。

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

def measure(n):
    "Measurement model, return two coupled measurements."
    m1 = np.random.normal(size=n)
    m2 = np.random.normal(scale=0.5, size=n)
    return m1+m2, m1-m2

m1, m2 = measure(2000)
xmin = m1.min()
xmax = m1.max()
ymin = m2.min()
ymax = m2.max()

X, Y = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
positions = np.vstack([X.ravel(), Y.ravel()])
values = np.vstack([m1, m2])
kernel = stats.gaussian_kde(values)
Z = np.reshape(kernel(positions).T, X.shape)

fig, ax = plt.subplots()
ax.imshow(np.rot90(Z), cmap=plt.cm.gist_earth_r,
          extent=[xmin, xmax, ymin, ymax])
ax.plot(m1, m2, 'k.', markersize=2)
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])

情节是：

现在，我们从从KDE获得的PDF中获取随机数据，这是变量Z。

# Generate the bins for each axis
x_bins = np.linspace(xmin, xmax, Z.shape[0]+1)
y_bins = np.linspace(ymin, ymax, Z.shape[1]+1)

# Find the middle point for each bin
x_bin_midpoints = x_bins[:-1] + np.diff(x_bins)/2
y_bin_midpoints = y_bins[:-1] + np.diff(y_bins)/2

# Calculate the Cumulative Distribution Function(CDF)from the PDF
cdf = np.cumsum(Z.ravel())
cdf = cdf / cdf[-1] # Normalização

# Create random data
values = np.random.rand(10000)

# Find the data position
value_bins = np.searchsorted(cdf, values)
x_idx, y_idx = np.unravel_index(value_bins,
                                (len(x_bin_midpoints),
                                 len(y_bin_midpoints)))

# Create the new data
new_data = np.column_stack((x_bin_midpoints[x_idx],
                            y_bin_midpoints[y_idx]))
new_x, new_y = new_data.T

我们可以根据这些新数据和情节计算KDE。

kernel = stats.gaussian_kde(new_data.T)
new_Z = np.reshape(kernel(positions).T, X.shape)

fig, ax = plt.subplots()
ax.imshow(np.rot90(new_Z), cmap=plt.cm.gist_earth_r,
          extent=[xmin, xmax, ymin, ymax])
ax.plot(new_x, new_y, 'k.', markersize=2)
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])

Answer 5

这是一个解决方案，它返回均匀分布在每个容器中而不是容器中心的数据点：

def draw_from_hist(hist, bins, nsamples = 100000):
    cumsum = [0] + list(I.np.cumsum(hist))
    rand = I.np.random.rand(nsamples)*max(cumsum)
    return [I.np.interp(x, cumsum, bins) for x in rand]

Answer 6

对于@ daniel，@ acro-bast，等

提出的解决方案，有些事情不能很好地解决

以最后一个示例

def draw_from_hist(hist, bins, nsamples = 100000):
    cumsum = [0] + list(I.np.cumsum(hist))
    rand = I.np.random.rand(nsamples)*max(cumsum)
    return [I.np.interp(x, cumsum, bins) for x in rand]

这假设至少第一个bin的内容为零，这可能是正确的，也可能不是。其次，这假设PDF的值位于垃圾箱的上边界，实际上不是，它位于垃圾箱的中心。

这是分为两个部分的另一种解决方案

def init_cdf(hist,bins):
    """Initialize CDF from histogram

    Parameters
    ----------
        hist : array-like, float of size N
            Histogram height 
        bins : array-like, float of size N+1
            Histogram bin boundaries 

    Returns:
    --------
        cdf : array-like, float of size N+1
    """
    from numpy import concatenate, diff,cumsum

    # Calculate half bin sizes
    steps  = diff(bins) / 2  # Half bin size

    # Calculate slope between bin centres 
    slopes = diff(hist) / (steps[:-1]+steps[1:]) 

    # Find height of end points by linear interpolation
    # - First part is linear interpolation from second over first
    #   point to lowest bin edge
    # - Second part is linear interpolation left neighbor to 
    #   right neighbor up to but not including last point
    # - Third part is linear interpolation from second to last point 
    #   over last point to highest bin edge
    # Can probably be done more elegant
    ends = concatenate(([hist[0] - steps[0] * slopes[0]], 
                        hist[:-1] + steps[:-1] * slopes,
                        [hist[-1] + steps[-1] * slopes[-1]]))

    # Calculate cumulative sum 
    sum = cumsum(ends)
    # Subtract off lower bound and scale by upper bound 
    sum -= sum[0]
    sum /= sum[-1]

    # Return the CDF 
    return sum

def sample_cdf(cdf,bins,size):
    """Sample a CDF defined at specific points.

    Linear interpolation between defined points 

    Parameters
    ----------
       cdf : array-like, float, size N
           CDF evaluated at all points of bins. First and 
           last point of bins are assumed to define the domain
           over which the CDF is normalized. 
       bins : array-like, float, size N
           Points where the CDF is evaluated.  First and last points 
           are assumed to define the end-points of the CDF's domain
       size : integer, non-zero
           Number of samples to draw 
    Returns
    -------
        sample : array-like, float, of size ``size``
             Random sample
    """
    from numpy import interp
    from numpy.random import random 

    return interp(random(size), cdf, bins)

# Begin example code
import numpy as np
import matplotlib.pyplot as plt

# initial histogram, coarse binning
hist,bins = np.histogram(np.random.normal(size=1000),np.linspace(-2,2,21))

# Calculate CDF, make sample, and new histogram w/finer binning
cdf = init_cdf(hist,bins)
sample = sample_cdf(cdf,bins,1000)
hist2,bins2 = np.histogram(sample,np.linspace(-3,3,61))

# Calculate bin centres and widths 
mx = (bins[1:]+bins[:-1])/2
dx = np.diff(bins)
mx2 = (bins2[1:]+bins2[:-1])/2
dx2 = np.diff(bins2)

# Plot, taking care to show uncertainties and so on
plt.errorbar(mx,hist/dx,np.sqrt(hist)/dx,dx/2,'.',label='original')
plt.errorbar(mx2,hist2/dx2,np.sqrt(hist2)/dx2,dx2/2,'.',label='new')
plt.legend()

对不起，我不知道如何使它显示在StackOverflow中，因此请复制粘贴并运行以查看要点。

Answer 7

当我寻找一种基于另一个数组的分布来生成随机数组的方法时，我偶然发现了这个问题。如果这是numpy，我将其称为random_like()函数。

然后我意识到，我编写了一个软件包Redistributor，即使该软件包的创建动机有所不同（Sklearn转换器能够将数据从任意分布转换为任意已知分布），该软件包仍可能为我做这件事用于机器学习）。当然，我知道不需要不必要的依赖关系，但至少知道这一包可能有一天对您有用。 OP询问的事情基本上是在这里进行的。

警告：在引擎盖下，所有操作均以一维完成。该程序包还实现了多维包装器，但是由于我觉得它太小众，所以我没有使用它编写此示例。

安装：

pip install git+https://gitlab.com/paloha/redistributor

实施：

import numpy as np
import matplotlib.pyplot as plt

def random_like(source, bins=0, seed=None):
    from redistributor import Redistributor
    np.random.seed(seed)
    noise = np.random.uniform(source.min(), source.max(), size=source.shape)
    s = Redistributor(bins=bins, bbox=[source.min(), source.max()]).fit(source.ravel())
    s.cdf, s.ppf = s.source_cdf, s.source_ppf
    r = Redistributor(target=s, bbox=[noise.min(), noise.max()]).fit(noise.ravel())
    return r.transform(noise.ravel()).reshape(noise.shape)

source = np.random.normal(loc=0, scale=1, size=(100,100))
t = random_like(source, bins=80) # More bins more precision (0 = automatic)

# Plotting
plt.figure(figsize=(12,4))
plt.subplot(121); plt.title(f'Distribution of source data, shape: {source.shape}')
plt.hist(source.ravel(), bins=100)
plt.subplot(122); plt.title(f'Distribution of generated data, shape: {t.shape}') 
plt.hist(t.ravel(), bins=100); plt.show()

说明：

import numpy as np
import matplotlib.pyplot as plt
from redistributor import Redistributor
from sklearn.metrics import mean_squared_error

# We have some source array with "some unknown" distribution (e.g. an image)
# For the sake of example we just generate a random gaussian matrix
source = np.random.normal(loc=0, scale=1, size=(100,100))
plt.figure(figsize=(12,4))
plt.subplot(121); plt.title('Source data'); plt.imshow(source, origin='lower') 
plt.subplot(122); plt.title('Source data hist'); plt.hist(source.ravel(), bins=100); plt.show()

# We want to generate a random matrix from the distribution of the source
# So we create a random uniformly distributed array called noise
noise = np.random.uniform(source.min(), source.max(), size=(100,100))
plt.figure(figsize=(12,4))
plt.subplot(121); plt.title('Uniform noise'); plt.imshow(noise, origin='lower')
plt.subplot(122); plt.title('Uniform noise hist'); plt.hist(noise.ravel(), bins=100); plt.show()

# Then we fit (approximate) the source distribution using Redistributor
# This step internally approximates the cdf and ppf functions.
s = Redistributor(bins=200, bbox=[source.min(), source.max()]).fit(source.ravel())

# A little naming workaround to make obj s work as a target distribution
s.cdf = s.source_cdf
s.ppf = s.source_ppf

# Here we create another Redistributor but now we use the fitted Redistributor s as a target
r = Redistributor(target=s, bbox=[noise.min(), noise.max()])

# Here we fit the Redistributor r to the noise array's distribution
r.fit(noise.ravel())

# And finally, we transform the noise into the source's distribution
t = r.transform(noise.ravel()).reshape(noise.shape)
plt.figure(figsize=(12,4))
plt.subplot(121); plt.title('Transformed noise'); plt.imshow(t, origin='lower')
plt.subplot(122); plt.title('Transformed noise hist'); plt.hist(t.ravel(), bins=100); plt.show()

# Computing the difference between the two arrays
print('Mean Squared Error between source and transformed: ', mean_squared_error(source, t))

源与已转换之间的均方误差：2.0574123162302143