```
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
import torch
from torch.distributions.normal import Normal
from torch.autograd import Variable
from typing import List, Optional
class GaussianGradientBoosting:
def __init__(self,
learning_rate: float = 0.025,
max_depth: int = 1,
n_estimators: int =100):
self.learning_rate: float = learning_rate
self.max_depth: int = max_depth
self.n_estimators: int = n_estimators
self.init_mu: Optional[float] = None
self.mu_trees: List[DecisionTreeRegressor] = []
self.init_sigma: Optional[float] = None
self.sigma_trees: List[DecisionTreeRegressor] = []
self.is_trained: bool = False
def predict(self, X: np.array) -> np.array:
assert self.is_trained
mus = self._predict_mus(X).reshape(-1,1)
log_sigmas = np.exp(self._predict_log_sigmas(X).reshape(-1,1))
return np.concatenate([mus, log_sigmas], 1)
def _predict_raw(self, X: np.array) -> np.array:
assert self.is_trained
mus = self._predict_mus(X).reshape(-1,1)
log_sigmas = self._predict_log_sigmas(X).reshape(-1,1)
return np.concatenate([mus, log_sigmas], 1)
def fit(self, X: np.array, y: np.array) -> None:
self._fit_initial(y)
self.is_trained = True
for _ in range(self.n_estimators):
y_pred = self._predict_raw(X)
gradients = self._get_gradients(y, y_pred)
mu_tree = DecisionTreeRegressor(max_depth=self.max_depth)
mu_tree.fit(X, gradients[:,0])
self.mu_trees.append(mu_tree)
sigma_tree = DecisionTreeRegressor(max_depth=self.max_depth)
sigma_tree.fit(X, gradients[:,1])
self.sigma_trees.append(sigma_tree)
def _fit_initial(self, y: np.array) -> None:
assert not self.is_trained
self.init_mu = np.mean(y)
self.init_sigma = np.log(np.std(y))
def _get_gradients(self, y: np.array, y_pred: np.array) -> np.array:
y_torch = torch.tensor(y).float()
y_pred_torch = Variable(torch.tensor(y_pred).float(), requires_grad=True)
normal_dist = Normal(y_pred_torch[:,0], torch.exp(y_pred_torch[:,1])).log_prob(y_torch).sum()
normal_dist.backward()
return y_pred_torch.grad.numpy()
def _predict_mus(self, X: np.array) -> np.array:
output = np.zeros(len(X))
output += self.init_mu
for tree in self.mu_trees:
output += self.learning_rate * tree.predict(X)
return output
def _predict_log_sigmas(self, X: np.array) -> np.array:
output = np.zeros(len(X))
output += self.init_sigma
for tree in self.sigma_trees:
output += self.learning_rate * tree.predict(X)
return output
```