Implementing Core Machine Learning Algorithms in Python

Posted on Oct 17, 2025 in Computer Engineering

1. Perceptron Algorithm Implementation

The Perceptron is a foundational linear classification algorithm. This implementation uses NumPy for efficient vector operations.

Perceptron Class Definition

import numpy as np, random

class Perceptron:
    def __init__(self, eta=0.01, n_iter=50, random_state=1):
        self.eta, self.n_iter, self.random_state = eta, n_iter, random_state

    def fit(self, x, y):
        rgen = np.random.RandomState(self.random_state)
        self.w_ = rgen.normal(0, 0.01, 1 + x.shape[1])
        self.errors_ = []
        for _ in range(self.n_iter):
            errors = 0
            for xi, target in zip(x, y):
                update = self.eta * (target - self.predict(xi))
                self.w_[1:] += update * xi
                self.w_[0] += update
                errors += int(update != 0.0)
            self.errors_.append(errors)
        return self

    def net_input(self, x): 
        return np.dot(x, self.w_[1:]) + self.w_[0]

    def predict(self, x): 
        return np.where(self.net_input(x) >= 0.0, 1, -1)

Testing the Perceptron Model

x = np.array([[1,2,3],[4,5,6],[7,8,9]])
y = np.array([1,1,1])
pn = Perceptron(eta=0.1, n_iter=3, random_state=1)
pn.fit(x, y)
print("Trained weights:", pn.w_)
print("Errors during training", pn.errors_)

2. Adaline (Adaptive Linear Neuron) with Gradient Descent

The Adaline algorithm minimizes the cost function using gradient descent. This implementation includes tracking the cost over epochs for visualization.

AdalineGD Class Definition

import matplotlib.pyplot as plt

class AdalineGD:
    def __init__(self, eta=0.01, n_iter=50, random_state=1):
        self.eta, self.n_iter, self.random_state = eta, n_iter, random_state

    def fit(self, X, y):
        rgen = np.random.RandomState(self.random_state)
        self.w_ = rgen.normal(0.0, 0.01, 1 + X.shape[1])
        self.cost_ = []
        for _ in range(self.n_iter):
            output = self.activation(self.net_input(X))
            errors = y - output
            self.w_[1:] += self.eta * X.T.dot(errors)
            self.w_[0] += self.eta * errors.sum()
            self.cost_.append((errors**2).sum() / 2.0)
        return self

    def net_input(self, X): 
        return np.dot(X, self.w_[1:]) + self.w_[0]

    def activation(self, X): 
        return X

    def predict(self, X): 
        return np.where(self.activation(self.net_input(X)) >= 0.0, 1, -1)

Training and Visualization

ad = AdalineGD(eta=0.1, n_iter=3, random_state=1)
ad.fit(x, y)
print("weight:", ad.w_)
print("cost:", ad.cost_)

plt.plot(ad.cost_); 
plt.title("Adaline Cost"); 
plt.xlabel("Epochs"); 
plt.ylabel("Cost"); 
plt.show()

plt.plot(ad.w_); 
plt.title("Weights"); 
plt.xlabel("Index"); 
plt.ylabel("Weight"); 
plt.show()

3. Logistic Regression for Wine Classification using PCA

This section demonstrates using Logistic Regression from Scikit-learn for multi-class classification on the Wine dataset, utilizing Principal Component Analysis (PCA) for dimensionality reduction and visualization.

Data Preparation and Model Setup

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.decomposition import PCA
from sklearn.metrics import classification_report, accuracy_score, confusion_matrix
from matplotlib.colors import ListedColormap

url = "https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data"
df = pd.read_csv(url, header=None)

# Column names definition (corrected 'Alcalinity' spelling)
df.columns = ['Class label','Alcohol','Malic acid','Ash','Alkalinity of ash','Magnesium','Total phenols','Flavanoids','Nonflavanoid phenols','Proanthocyanins','Color intensity','Hue','OD280/OD315 of diluted wines','Proline']

X, y = df.iloc[:, 1:].values, df.iloc[:, 0].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y, random_state=0)

# Standardize features
sc = StandardScaler()
X_train_std, X_test_std = sc.fit_transform(X_train), sc.transform(X_test)

# Apply PCA to reduce dimensions to 2 components
pca = PCA(n_components=2)
X_train_pca, X_test_pca = pca.fit_transform(X_train_std), pca.transform(X_test_std)

# Train Logistic Regression model
lr = LogisticRegression(max_iter=10000)
lr.fit(X_train_pca, y_train)

Decision Region Plotting Function

def plot_decision_regions(X, y, classifier, res=0.02):
    markers, colors = ('s','x','o','^','v'), ('red','blue','lightgreen','gray','cyan')
    cmap = ListedColormap(colors[:len(np.unique(y))])
    
    x1_min, x1_max = X[:,0].min()-1, X[:,0].max()+1
    x2_min, x2_max = X[:,1].min()-1, X[:,1].max()+1
    
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, res), np.arange(x2_min, x2_max, res))
    
    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T).reshape(xx1.shape)
    
    plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)
    
    for idx, cl in enumerate(np.unique(y)):
        plt.scatter(X[y==cl, 0], X[y==cl, 1], alpha=0.6, c=[cmap(idx)], marker=markers[idx], label=cl)

Visualization and Evaluation

plot_decision_regions(X_train_pca, y_train, classifier=lr)
plt.xlabel('PC1'); 
plt.ylabel('PC2'); 
plt.legend(loc='lower left'); 
plt.title('Logistic Regression Decision Regions'); 
plt.show()

# Evaluate performance on the test set
y_pred = lr.predict(X_test_pca)
print("Accuracy:", accuracy_score(y_test, y_pred))
print("Confusion Matrix:\n", confusion_matrix(y_test, y_pred))
print("Classification Report:\n", classification_report(y_test, y_pred))