Lesson 17-Advanced Algorithm Development

Custom Optimization Algorithm Implementation

Genetic Algorithm (GA) Advanced Applications

Multi-Objective Optimization Problem Solving:

% Define multi-objective optimization problem
function f = multiObjective(x)
    f(1) = x(1)^2 + x(2)^2;       % Objective 1: Minimize distance to origin
    f(2) = (x(1)-1)^2 + (x(2)-1)^2; % Objective 2: Minimize distance to point (1,1)
end

% Configure genetic algorithm options
options = optimoptions('gamultiobj',...
    'PopulationSize', 100,...
    'MaxGenerations', 200,...
    'CrossoverFraction', 0.8,...
    'ParetoFraction', 0.35,...
    'Display', 'iter',...
    'PlotFcn', {@gaplotpareto,@gaplotscorediversity});

% Run genetic algorithm
nvars = 2; % Variable dimensions
lb = [0 0]; % Lower bound
ub = [1 1]; % Upper bound
[x, fval, exitflag, output, population] = gamultiobj(@multiObjective, nvars, [], [], [], [], lb, ub, [], options);

% Result visualization
figure;
scatter(fval(:,1), fval(:,2), 'filled');
xlabel('Objective 1');
ylabel('Objective 2');
title('Pareto Front Solution Set');
grid on;

Hybrid Genetic Algorithm Design:

% Genetic algorithm combined with local search
function [x, fval] = hybridGA(fun, nvars, lb, ub)
    options = optimoptions('gamultiobj',...
        'PopulationSize', 50,...
        'MaxGenerations', 100,...
        'UseVectorized', false);
    
    % Initial genetic algorithm optimization
    [x_ga, fval_ga] = gamultiobj(fun, nvars, [], [], [], [], lb, ub, options);
    
    % Perform local search on Pareto front solutions
    options_ls = optimoptions('fmincon',...
        'Algorithm', 'sqp',...
        'Display', 'off');
    
    x_final = x_ga;
    fval_final = fval_ga;
    for i = 1:size(x_ga,1)
        [x_local, fval_local] = fmincon(fun, x_ga(i,:), [], [], [], [], lb, ub, [], options_ls);
        if dominates(fval_local, fval_final(i,:))
            x_final(i,:) = x_local;
            fval_final(i,:) = fval_local;
        end
    end
    
    x = x_final;
    fval = fval_final;
end

% Dominance relation judgment function
function d = dominates(f1, f2)
    d = all(f1 <= f2) && any(f1 < f2);
end

Simulated Annealing Algorithm Customization

Discrete Optimization Problem Solving:

% Traveling Salesman Problem (TSP) solving
function [bestRoute, minDist] = saTSP(distMatrix)
    nCities = size(distMatrix,1);
    
    % Initial solution (random path)
    currentRoute = randperm(nCities);
    currentDist = calculateDistance(currentRoute, distMatrix);
    
    % Simulated annealing parameters
    T0 = 100;       % Initial temperature
    Tend = 1e-3;    % Termination temperature
    alpha = 0.95;   % Cooling coefficient
    maxIter = 1000; % Iterations per temperature
    
    % Initialization
    bestRoute = currentRoute;
    minDist = currentDist;
    T = T0;
    
    % Main loop
    while T > Tend
        for i = 1:maxIter
            % Generate neighborhood solution (swap two cities)
            newRoute = currentRoute;
            idx = randperm(nCities,2);
            newRoute(idx) = newRoute(fliplr(idx));
            
            % Calculate new solution distance
            newDist = calculateDistance(newRoute, distMatrix);
            
            % Acceptance criterion
            delta = newDist - currentDist;
            if delta < 0 || exp(-delta/T) > rand
                currentRoute = newRoute;
                currentDist = newDist;
                
                % Update optimal solution
                if currentDist < minDist
                    bestRoute = currentRoute;
                    minDist = currentDist;
                end
            end
        end
        % Cooling
        T = alpha * T;
    end
end

% Distance calculation function
function dist = calculateDistance(route, distMatrix)
    dist = 0;
    n = length(route);
    for i = 1:n-1
        dist = dist + distMatrix(route(i), route(i+1));
    end
    dist = dist + distMatrix(route(end), route(1)); % Closed path
end

Deep Learning Model Customization

Custom Network Layer Implementation

Attention Mechanism Layer Customization:

classdef AttentionLayer < nnet.layer.Layer
    properties
        NumHeads
        KeyDim
        LearnableWeights
    end
    
    methods
        function layer = AttentionLayer(numHeads, keyDim)
            % Constructor
            arguments
                numHeads (1,1) {mustBePositive, mustBeInteger}
                keyDim (1,1) {mustBePositive, mustBeInteger}
            end
            layer.NumHeads = numHeads;
            layer.KeyDim = keyDim;
            layer.Description = "Multi-head Attention Layer";
        end
        
        function [Z, memory] = predict(layer, X)
            % Forward propagation (inference mode)
            [Q, K, V] = computeQKV(layer, X);
            Z = scaledDotProductAttention(Q, K, V, layer.NumHeads);
        end
        
        function [Z, memory, Z_grad, Q_grad, K_grad, V_grad] = forward(layer, X, ~)
            % Forward propagation (training mode)
            [Q, K, V, Q_grad, K_grad, V_grad] = computeQKV(layer, X);
            [Z, Z_grad] = scaledDotProductAttention(Q, K, V, layer.NumHeads);
        end
        
        function [dLdX, dLdW] = backward(layer, X, ~, dLdZ)
            % Backward propagation
            [Q, K, V, Q_grad, K_grad, V_grad] = computeQKV(layer, X);
            [dLdQ, dLdK, dLdV] = backwardScaledDotProductAttention(...
                Q, K, V, dLdZ, layer.NumHeads);
            
            dLdX = Q_grad * dLdQ + K_grad * dLdK + V_grad * dLdV;
        end
        
        function [Q, K, V] = computeQKV(layer, X)
            % Compute query, key, value matrices
            [batchSize, seqLen, inputDim] = size(X);
            projDim = layer.NumHeads * layer.KeyDim;
            
            % Learnable parameters
            if isempty(layer.LearnableWeights)
                layer.LearnableWeights = [
                    initializeWeights([inputDim, projDim]), ...
                    initializeWeights([inputDim, projDim]), ...
                    initializeWeights([inputDim, projDim])
                ];
            end
            
            WQ = layer.LearnableWeights{1};
            WK = layer.LearnableWeights{2};
            WV = layer.LearnableWeights{3};
            
            Q = pagemtimes(X, WQ); % batchSize x seqLen x projDim
            K = pagemtimes(X, WK);
            V = pagemtimes(X, WV);
            
            % Split heads
            Q = reshape(Q, batchSize, seqLen, layer.NumHeads, layer.KeyDim);
            K = reshape(K, batchSize, seqLen, layer.NumHeads, layer.KeyDim);
            V = reshape(V, batchSize, seqLen, layer.NumHeads, layer.KeyDim);
            
            Q = permute(Q, [1 3 2 4]); % batchSize x NumHeads x seqLen x KeyDim
            K = permute(K, [1 3 2 4]);
            V = permute(V, [1 3 2 4]);
        end
    end
end

% Auxiliary functions
function weights = initializeWeights(dim)
    weights = randn(dim) * sqrt(2/dim(1)); % He initialization
end

function Z = scaledDotProductAttention(Q, K, V, numHeads)
    % Scaled dot-product attention
    dK = size(K,4);
    scores = pagemtimes(Q, permute(K, [1 2 4 3])) / sqrt(dK);
    attentionWeights = softmax(scores, 3);
    Z = pagemtimes(attentionWeights, V);
    Z = permute(Z, [1 3 2 4]);
    Z = reshape(Z, size(Z,1), size(Z,2), numHeads*dK);
end

Custom Loss Functions

Focal Loss Implementation:

classdef FocalLossLayer < nnet.layer.ClassificationLayer
    properties
        Alpha = 0.25
        Gamma = 2
    end
    
    methods
        function layer = FocalLossLayer(name)
            layer.Name = name;
            layer.Description = "Focal Loss Layer";
        end
        
        function loss = forwardLoss(layer, Y, T)
            % Y: predictions (numClasses x numObservations)
            % T: targets (categorical or one-hot)
            
            if iscategorical(T)
                T = onehotencode(T, 1);
            end
            
            p = Y .* T + (1 - Y) .* (1 - T);
            modulatingFactor = (1 - p) .^ layer.Gamma;
            ceLoss = -T .* log(Y) - (1 - T) .* log(1 - Y);
            focalLoss = modulatingFactor .* ceLoss;
            
            % Alpha weighting
            alphaWeight = layer.Alpha * T + (1 - layer.Alpha) * (1 - T);
            focalLoss = alphaWeight .* focalLoss;
            
            loss = sum(focalLoss(:)) / size(Y, 2);
        end
        
        function dLdY = backwardLoss(layer, Y, T)
            % Compute gradient w.r.t predictions
            if iscategorical(T)
                T = onehotencode(T, 1);
            end
            
            p = Y .* T + (1 - Y) .* (1 - T);
            modFactor = (1 - p) .^ layer.Gamma;
            d_ce = -T ./ Y + (1 - T) ./ (1 - Y);
            
            d_mod = layer.Gamma * (1 - p) .^ (layer.Gamma - 1) .* log(p) .* (T - Y);
            
            alphaWeight = layer.Alpha * T + (1 - layer.Alpha) * (1 - T);
            
            dLdY = alphaWeight .* (modFactor .* d_ce + d_mod) / size(Y, 2);
        end
    end
end

Fixed-Point Arithmetic Implementation

Fixed-Point Filter Design

Fixed-Point FIR Filter:

% Design floating-point FIR filter
fs = 1000; % Sampling frequency
fc = 200;  % Cutoff frequency
order = 30;
b = fir1(order, fc/(fs/2));

% Convert to fixed-point
fi_b = fi(b, 1, 16, 15); % 16-bit signed, 15 fractional bits

% Fixed-point filtering
x = randn(1000,1); % Input signal
y_float = filter(b, 1, x);
y_fixed = filter(fi_b, 1, x);

% Error analysis
error = y_float - double(y_fixed);
snr = 10*log10(mean(x.^2)/mean(error.^2));
fprintf('Fixed-point SNR: %.2f dB\n', snr);

% Quantization noise modeling
quantNoise = randn(size(x)) * 2^(-15);
y_noisy = filter(b, 1, x + quantNoise);

Real-Time Signal Processing

Real-Time FIR Filter Implementation:

% Fixed-point coefficient quantization
h = fir1(30, 0.4); % Floating-point coefficients
h_fixed = fi(h, 1, 16, 14); % 16-bit fixed-point, 14 fractional bits

% Real-time filtering implementation (circular buffer)
classdef RealTimeFIRFilter
    properties
        Coefficients
        Buffer
        BufferIndex
        DelayLine
    end
    
    methods
        function obj = RealTimeFIRFilter(coefficients)
            obj.Coefficients = coefficients;
            obj.Buffer = zeros(size(coefficients));
            obj.BufferIndex = 1;
        end
        
        function [output, obj] = step(obj, inputSample)
            % Update delay line
            obj.Buffer(obj.BufferIndex) = inputSample;
            
            % Compute filter output
            output = 0;
            for i = 1:length(obj.Coefficients)
                idx = mod(obj.BufferIndex - i + length(obj.Buffer), length(obj.Buffer)) + 1;
                output = output + obj.Buffer(idx) * obj.Coefficients(i);
            end
            
            % Update buffer index
            obj.BufferIndex = mod(obj.BufferIndex, length(obj.Buffer)) + 1;
        end
    end
end

% Usage example
firFilter = RealTimeFIRFilter(h_fixed);
inputSignal = randn(1e6,1); % Simulate input signal
outputSignal = zeros(size(inputSignal));

for i = 1:length(inputSignal)
    [outputSignal(i), firFilter] = firFilter.step(inputSignal(i));
end

Multi-Rate Signal Processing

CIC Filter Implementation:

classdef CICDecimator
    properties
        R           % Decimation factor
        M           % Differential delay
        N           % Number of stages
        Integrator  % Integrator states
        Comb        % Comb filter states
    end
    
    methods
        function obj = CICDecimator(R, M, N)
            arguments
                R (1,1) {mustBeInteger, mustBePositive}
                M (1,1) {mustBeInteger, mustBePositive}
                N (1,1) {mustBeInteger, mustBePositive}
            end
            obj.R = R;
            obj.M = M;
            obj.N = N;
            obj.Integrator = zeros(N,1);
            obj.Comb = zeros(N,1);
        end
        
        function [output, obj] = step(obj, inputSample)
            % Integrator part
            obj.Integrator = obj.Integrator + inputSample;
            
            % Decimation (process every R samples)
            if mod(obj.R,1) == 0 % Simple example, actual needs more complex control
                % Comb filter part
                diffOutput = obj.Integrator - obj.Comb;
                obj.Comb = obj.Integrator;
                
                % Multi-stage processing
                output = diffOutput;
                for i = 2:obj.N
                    output = output - circshift(obj.Comb, [0 i-1]);
                end
            else
                output = 0; % Non-decimation period outputs 0
            end
        end
    end
end

High-Dimensional Numerical Integration

Triple Integral Implementation

Triple Integral Over Complex Regions:

% Define integrand function
f = @(x,y,z) sin(x).*cos(y).*exp(z);

% Define integration region (example: unit ball)
gfun = @(x,y,z) x.^2 + y.^2 + z.^2 - 1; % g(x,y,z)<=0 defines region

% Perform integration using integral3
opts = optimoptions('integral3', 'AbsTol', 1e-6, 'RelTol', 1e-4);
q = integral3(f, -1, 1, -1, 1, -1, 1, ...
    'Method', 'tiled', ...
    'AbsTol', opts.AbsTol, ...
    'RelTol', opts.RelTol);

% Custom region integration (using implicit function definition)
q_custom = integral3(f, -1, 1, -1, 1, -1, 1, ...
    'Method', 'iterated', ...
    'AbsTol', opts.AbsTol, ...
    'RelTol', opts.RelTol, ...
    'FunValCheck', 'on');

% Verify integration result
exact_value = 4*pi/3 * (1 - cos(1)*sin(1)*exp(1)); % Analytical solution (if exists)
error = abs(q - exact_value);
fprintf('Integration error: %.4e\n', error);

High-Dimensional Adaptive Integration

Cubature Toolbox Application:

% Install cubature toolbox (pre-installed required)
% Add path
addpath('cubature');

% Define integrand function (high-dimensional example)
f = @(x) prod(sin(x)); % n-dimensional sin product function

% Integration dimensions
n = 5;
dim = n;

% Integration limits (hypercube)
xmin = zeros(1,dim);
xmax = ones(1,dim);

% Call cubature function
[integralValue, errorEstimate] = cubature(f, dim, xmin, xmax, ...
    'maxEval', 1e6, ...
    'absTol', 1e-6, ...
    'relTol', 1e-4);

fprintf('High-dimensional integration result: %.8f ± %.2e\n', integralValue, errorEstimate);

Distributed Machine Learning

Parallel Training Framework

Parameter Server Architecture Implementation:

% Master node (parameter server)
function parameterServer()
    % Initialize model parameters
    numWorkers = 4;
    params = initializeParameters();
    
    % Create shared memory area
    sharedParams = parallel.pool.Constant(params);
    
    % Start worker nodes
    parpool('local', numWorkers);
    spmd
        % Each worker node gets parameter copy
        localParams = sharedParams.Value;
        
        % Local data partitioning
        dataPartition = getDataPartition(labindex, numWorkers);
        
        % Training loop
        for epoch = 1:numEpochs
            % Compute local gradients
            gradients = computeGradients(localParams, dataPartition);
            
            % Asynchronous parameter update (delayed update)
            if mod(labindex, 2) == 0 % Partial nodes delayed update
                pause(rand*0.1); % Simulate network delay
            end
            
            % Parameter server aggregates gradients
            sharedParams.Value = aggregateGradients(sharedParams.Value, gradients);
            
            % Get latest parameters
            localParams = sharedParams.Value;
        end
    end
end

% Worker node function
function workerNode()
    % Get parameters from parameter server
    params = getParametersFromServer();
    
    % Local training
    while ~trainingComplete
        % Compute gradients
        gradients = computeLocalGradients(params);
        
        % Send gradients to parameter server
        sendGradientsToServer(gradients);
        
        % Get updated parameters
        params = getParametersFromServer();
    end
end

% Parameter aggregation function (asynchronous SGD)
function globalParams = aggregateGradients(globalParams, gradients)
    % Simple weighted average (actual needs more complex strategy)
    globalParams = globalParams - learningRate * gradients;
end

Data Parallel Training

Multi-GPU Training Implementation:

% Multi-GPU data parallel training
function trainMultiGPU()
    % Detect number of available GPUs
    numGPUs = gpuDeviceCount();
    if numGPUs < 2
        error('At least 2 GPUs required');
    end
    
    % Create GPU device array
    gpus = repmat(arrayfun(@(x) gpuDevice(x), 1:numGPUs, 'UniformOutput', false), 1);
    
    % Data partitioning
    data = loadData();
    partitions = partitionData(data, numGPUs);
    
    % Model replication
    models = cell(1, numGPUs);
    for i = 1:numGPUs
        models{i} = createModel();
        models{i} = moveModelToGPU(models{i}, gpus{i});
    end
    
    % Training loop
    for epoch = 1:numEpochs
        % Parallel training
        parfor i = 1:numGPUs
            % Get current partition data
            batch = getNextBatch(partitions{i});
            
            % Forward propagation
            [loss, gradients] = forwardBackward(models{i}, batch);
            
            % Update model parameters
            models{i} = updateModel(models{i}, gradients);
        end
        
        % Model synchronization (average parameters)
        models{1} = synchronizeModels(models);
    end
end

% Model synchronization function
function syncedModel = synchronizeModels(models)
    % Simple average (actual needs more efficient communication strategy)
    params = getParameters(models{1});
    for i = 2:length(models)
        params = params + getParameters(models{i});
    end
    params = params / length(models);
    syncedModel = setParameters(models{1}, params);
end
Share your love