Law of Large Numbers and Central Limit Theorem
Law of Large Numbers and Central Limit Theorem
Samia Challal
19/05/2021
1
Law of Large numbers.
If X1, X2, ..., Xn, ... is a sequence of unco
elated random variables, each with mu and variance σ2, then
lim
n→+∞
P (|Xn − µ| ⩾ δ) = 0 where Xn =
X XXXXXXXXXXXn
n
That is Xn approaches the population mean µ as the sample size n increases in the sense of the probability
measure:
(X XXXXXXXXXXXn)/n −→ µ
Central limit theorem.
Let X1, X2, . . . be an infinite sequence of independent random variables; each with the same distribution
f(x), and finite mean µ, and variance σ2. For any numbers a and
lim
n→+∞
P
(
a ⩽
X − µ
σ
√
n
⩽
)
= 1√
2π
∫
a
e−t
2/2dt
where X = (X1 + X XXXXXXXXXXXn
n
.
Application.
If Z ⇝ N(0, 1), then
lim
n→+∞
P
(
a ⩽
X − µ
σ
√
n
⩽
)
= lim
n→+∞
P
(
a ⩽ Z ⩽
)
; X − µ
σ
√
n
≈⇝ N(0, 1).
We would like to illustrate these two theorems for different distributions.
2
Case of Binomial distributions: Xi are (B(1,p))
Then
X = X XXXXXXXXXXXn has a B(n, p) distribution
p= 0.2 # probability of success = mean of the population
mu=p
s0=sqrt(p*(1-p)) # standard deviation of the population
n1=10 # Number of trials
n2=20
n3=50
n4=200
s1 <- c()
s2 <- c()
s3 <- c()
s4 <- c()
ows =1000 # Number of simulations
for ( i in 1:rows){
# gerenates nk, k=1,2,3,4 numbers from B(1,p) and take their mean
s1[i] = mean(
inom(n = n1, size = 1, prob = p))
s2[i] = mean(
inom(n = n2, size = 1, prob = p))
s3[i] = mean(
inom(n = n3, size = 1, prob = p))
s4[i] = mean(
inom(n = n4, size = 1, prob = p))
}
# prepare like a matrix of 2 rows and 2 columns where the graphs will be inserted in each cell
par(mfrow=c(2,2))
# plot the histogram of the relative frequency of the sample of means s1 where each size sample is n1.
hist(s1, col ="lightblue",main=paste("Sample size=", n1),
xlab ="observed means",ylab="Probability", freq=FALSE, probability=TRUE)
# plot the vertical line x=p=mean
abline(v = mu, col = "red",lwd="2")
# plot the normal density function N(mu, s0/sqrt(n1))
curve(dnorm(x, mean = mu, sd=s0/sqrt(n1)),
col="green",
lwd="2",
add=T)
3
hist(s2, col ="lightblue",main=paste("Sample size=", n2),
xlab ="observed means",ylab="Probability", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n2)),
col="green",
lwd="2",
add=T)
hist(s3, col ="lightblue",main=paste("Sample size=", n3),
xlab ="observed means",ylab="Probability", freq=FALSE )
abline(v =mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n3)),
col="green",
lwd="2",
add=T)
hist(s4, col ="lightblue",main=paste("Sample size=", n4),
xlab ="observed means",ylab="Probability", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n4)),
col="green",
lwd="2",
add=T)
4
Sample size= 10
observed means
P
o
a
i
lit
y
XXXXXXXXXX0.5 0.6
0
2
4
6
Sample size= 20
observed means
P
o
a
i
lit
y
XXXXXXXXXX0.5
0
2
4
Sample size= 50
observed means
P
o
a
i
lit
y
XXXXXXXXXX
0
4
8
Sample size= 200
observed means
P
o
a
i
lit
y
XXXXXXXXXX
0
4
8
The vertical line is set at the position p = 0.02, the mean of the population.
As n increases, the data of means clusters around p which illustrates well the theorem.
This shows also that (X XXXXXXXXXXXn)/n is a good point estimate for µ = p.
Another, remark, is the mound shape of the distribution.
5
Case of Normal distributions: Xi are N(0, 1)
Then
(X XXXXXXXXXXXn)/n has a N(0, 1
√
n) distribution
mu=0
s= 1 # Standard deviation
n1=10 # Number of trials
n2=20
n3=50
n4=100
s1 <- c()
s2 <- c()
s3 <- c()
s4 <- c()
ows =1000 # Number of simulations
for ( i in 1:rows){
s1[i] = mean(rnorm(n = n1, mean = 0, sd= s)) # gerenates n1 numbers
s2[i] = mean(rnorm(n = n2, mean = 0, sd= s))
s3[i] = mean(rnorm(n = n3, mean = 0, sd= s))
s4[i] = mean(rnorm(n = n4, mean = 0, sd= s))
}
par(mfrow=c(2,2))
hist(s1, col ="lightblue",main=paste("Sample size=", n1),
xlab ="observed means",ylab="Probability", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s/sqrt(n1)),
col="green",
lwd="2",
add=T)
hist(s2, col ="lightblue",main=paste("Sample size=", n2),
xlab ="observed means",ylab="Probability", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s/sqrt(n2)),
col="green",
lwd="2",
add=T)
hist(s3, col ="lightblue",main=paste("Sample size=", n3),
xlab ="observed means",ylab="Probability", freq=FALSE )
6
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s/sqrt(n3)),
col="green",
lwd="2",
add=T)
hist(s4, col ="lightblue",main=paste("Sample size=", n4),
xlab ="observed means",ylab="Probability", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s/sqrt(n4)),
col="green",
lwd="2",
add=T)
Sample size= 10
observed means
P
o
a
i
lit
y
−1.0 − XXXXXXXXXX
0.
0
0.
6
1.
2
Sample size= 20
observed means
P
o
a
i
lit
y
− XXXXXXXXXX
0.
0
1.
0
Sample size= 50
observed means
P
o
a
i
lit
y
−0.4 − XXXXXXXXXX
0.
0
1.
5
Sample size= 100
observed means
P
o
a
i
lit
y
− XXXXXXXXXX
0
2
4
7
Case of Poisson distributions: Xi are P (λ)
mu=0.2 # p=lambda=mean
s0=mu # s0=standard deviation
n1=10 # Number of trials
n2=20
n3=50
n4=100
s1 <- c()
s2 <- c()
s3 <- c()
s4 <- c()
ows =1000 # Number of simulations
for ( i in 1:rows){
s1[i] = mean(rpois(n = n1, lambda=mu)) # gerenates n1 numbers
s2[i] = mean(rpois(n = n2, lambda=mu))
s3[i] = mean(rpois(n = n3, lambda=mu))
s4[i] = mean(rpois(n = n4, lambda=mu))
}
par(mfrow=c(2,2))
hist(s1, col ="greenyellow",main=paste("Sample size=", n1),
xlab ="observed means",ylab="Probability", freq=FALSE , xlim=c(0,1))
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n1)),
col="blue",
lwd="2",
add=T)
hist(s2, col ="greenyellow",main=paste("Sample size=", n2),
xlab ="observed means",ylab="Probability", freq=FALSE , xlim=c(0,1))
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n2)),
col="blue",
lwd="2",
add=T)
hist(s3, col ="greenyellow",main=paste("Sample size=", n3),
xlab ="observed means",ylab="Probability", freq=FALSE , xlim=c(0,1))
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n3)),
col="blue",
lwd="2",
add=T)
8
hist(s4, col ="greenyellow",main=paste("Sample size=", n4),
xlab ="observed means",ylab="Probability", freq=FALSE , xlim=c(0,1))
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n4)),
col="blue",
lwd="2",
add=T)
Sample size= 10
observed means
P
o
a
i
lit
y
XXXXXXXXXX1.0
0
2
4
Sample size= 20
observed means
P
o
a
i
lit
y
XXXXXXXXXX1.0
0
2
4
Sample size= 50
observed means
P
o
a
i
lit
y
XXXXXXXXXX1.0
0
4
Sample size= 100
observed means
P
o
a
i
lit
y
XXXXXXXXXX1.0
0
4
8
9
Case of Discrete Uniform distributions: Xi are U(x1, x2, . . . , xn)
mu=mean(0:4) # mu = mean of the population
s0=sd(0:4) # s0= standard deviatio of the population
n1=10 # Number of trials
n2=20
n3=50
n4=100
s1 <- c()
s2 <- c()
s3 <- c()
s4 <- c()
ows =1000 # Number of simulations
for ( i in 1:rows){
s1[i] = mean(sample(0:4, n1, replace = TRUE)) # gerenates n1 numbers
s2[i] = mean(sample(0:4, n2, replace = TRUE))
s3[i] = mean(sample(0:4, n3, replace = TRUE))
s4[i] = mean(sample(0:4, n4, replace = TRUE))
}
par(mfrow=c(2,2))
hist(s1, col ="greenyellow",main=paste("Sample size=", n1),
xlab ="observed means",ylab="Relative Frequency", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n1)),
col="blue",
lwd="2",
add=T)
hist(s2, col ="greenyellow",main=paste("Sample size=", n2),
xlab ="observed means",ylab="Relative Frequency", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n2)),
col="blue",
lwd="2",
add=T)
hist(s3, col ="greenyellow",main=paste("Sample size=", n3),
xlab ="observed means",ylab="Relative Frequency", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n3)),
col="blue",
lwd="2",
add=T)
10
hist(s4, col ="greenyellow",main=paste("Sample size=", n4),
xlab ="observed means",ylab="Relative Frequency", freq=FALSE )
abline(v = mu, col = "red",lwd="2")
curve(dnorm(x, mean = mu, sd=s0/sqrt(n4)),
col="blue",
lwd="2",
add=T)
Sample size= 10
observed means
R
el
at
iv
e
F
e
qu
en
cy
XXXXXXXXXX3.0 3.5
0.
0
0.
6
Sample size= 20
observed means
R
el
at
iv
e
F
e
qu
en
cy
XXXXXXXXXX
0.
0
0.
6
1.
2
Sample size= 50
observed means
R
el
at
iv
e
F
e
qu
en
cy
XXXXXXXXXX
0.
0
1.
0
2.
0
Sample size= 100
observed means
R
el
at
iv
e
F
e
qu
en
cy
XXXXXXXXXX
0.
0
1.
5
11
Law of Large numbers.
Central limit theorem.
Application.
Case of Binomial distributions: X_i are (B(1,p))
Case of Normal distributions: X_i are N(0,1)
Case of Poisson distributions: X_i are P(\lambda)
Case of Discrete Uniform distributions: X_i are U(x_1, x_2, \ldots, x_n)