李伟
319 lines
12 KiB
C#
319 lines
12 KiB
C#
// ============================================================================
|
|
// Copyright 穢 2026 Hexagon Technology Center GmbH. All Rights Reserved.
|
|
// ��辣�? SuperResolutionProcessor.cs
|
|
// �讛膩: �箔�瘛勗漲摮虫������儘���摮?
|
|
// �蠘�:
|
|
// - �舀� EDSR �?FSRCNN 頞��颲函�璅∪�嚗㇉NNX �澆�嚗?
|
|
// - �舀� 2x�?x�?x �曉之�滨�
|
|
// - �啣漲�曉��芸𢆡頧祆揢銝箔��𡁻�颲枏�嚗峕綫���頧砍��啣漲
|
|
// - 璅∪���辣�芸𢆡�𦦵揣嚗峕𣈲��䌊摰帋�頝臬�
|
|
// - 雿輻鍂 Microsoft.ML.OnnxRuntime 餈𥡝��函�
|
|
// 蝞埈�: EDSR (Enhanced Deep Residual SR) / FSRCNN (Fast SR CNN)
|
|
// 雿𡏭�? �𦒘� wei.lw.li@hexagon.com
|
|
// ============================================================================
|
|
|
|
using Emgu.CV;
|
|
using Emgu.CV.CvEnum;
|
|
using Emgu.CV.Structure;
|
|
using Microsoft.ML.OnnxRuntime;
|
|
using Microsoft.ML.OnnxRuntime.Tensors;
|
|
using Serilog;
|
|
using XP.ImageProcessing.Core;
|
|
|
|
namespace XP.ImageProcessing.Processors;
|
|
|
|
/// <summary>
|
|
/// �箔�瘛勗漲摮虫������儘���摮琜�EDSR / FSRCNN嚗㚁�雿輻鍂 ONNX Runtime �函�
|
|
/// </summary>
|
|
public class SuperResolutionProcessor : ImageProcessorBase
|
|
{
|
|
private static readonly ILogger _logger = Log.ForContext<SuperResolutionProcessor>();
|
|
|
|
// 隡朞�蝻枏�嚗屸��漤�憭滚�頧?
|
|
private static InferenceSession? _cachedSession;
|
|
|
|
private static string _cachedModelKey = string.Empty;
|
|
|
|
public SuperResolutionProcessor()
|
|
{
|
|
Name = LocalizationHelper.GetString("SuperResolutionProcessor_Name");
|
|
Description = LocalizationHelper.GetString("SuperResolutionProcessor_Description");
|
|
}
|
|
|
|
protected override void InitializeParameters()
|
|
{
|
|
Parameters.Add("Model", new ProcessorParameter(
|
|
"Model",
|
|
LocalizationHelper.GetString("SuperResolutionProcessor_Model"),
|
|
typeof(string),
|
|
"FSRCNN",
|
|
null,
|
|
null,
|
|
LocalizationHelper.GetString("SuperResolutionProcessor_Model_Desc"),
|
|
new string[] { "EDSR", "FSRCNN" }));
|
|
|
|
Parameters.Add("Scale", new ProcessorParameter(
|
|
"Scale",
|
|
LocalizationHelper.GetString("SuperResolutionProcessor_Scale"),
|
|
typeof(string),
|
|
"2",
|
|
null,
|
|
null,
|
|
LocalizationHelper.GetString("SuperResolutionProcessor_Scale_Desc"),
|
|
new string[] { "2", "3", "4" }));
|
|
|
|
_logger.Debug("InitializeParameters");
|
|
}
|
|
|
|
public override Image<Gray, byte> Process(Image<Gray, byte> inputImage)
|
|
{
|
|
string model = GetParameter<string>("Model");
|
|
int scale = int.Parse(GetParameter<string>("Scale"));
|
|
|
|
// �交𪄳璅∪���辣
|
|
string modelPath = FindModelFile(model, scale);
|
|
if (string.IsNullOrEmpty(modelPath))
|
|
{
|
|
_logger.Error("Model file not found: {Model}_x{Scale}.onnx", model, scale);
|
|
throw new FileNotFoundException(
|
|
$"頞��颲函�璅∪���辣�芣𪄳�? {model}_x{scale}.onnx\n" +
|
|
$"霂瑕�璅∪���辣�曄蔭�唬誑銝衤遙銝��桀�:\n" +
|
|
$" 1. 蝔见��桀�/Models/\n" +
|
|
$" 2. 蝔见��桀�/\n" +
|
|
$"璅∪���閬?ONNX �澆���n" +
|
|
$"�臭蝙�?tf2onnx 隞?.pb 頧祆揢:\n" +
|
|
$" pip install tf2onnx\n" +
|
|
$" python -m tf2onnx.convert --input {model}_x{scale}.pb --output {model}_x{scale}.onnx --inputs input:0 --outputs output:0");
|
|
}
|
|
|
|
// �㰘蝸�硋��其�霂?
|
|
string modelKey = $"{model}_{scale}";
|
|
InferenceSession session;
|
|
if (_cachedModelKey == modelKey && _cachedSession != null)
|
|
{
|
|
session = _cachedSession;
|
|
_logger.Debug("Reusing cached session: {ModelKey}", modelKey);
|
|
}
|
|
else
|
|
{
|
|
_cachedSession?.Dispose();
|
|
var options = new SessionOptions();
|
|
options.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL;
|
|
try
|
|
{
|
|
options.AppendExecutionProvider_CUDA(0);
|
|
_logger.Information("Using CUDA GPU for inference");
|
|
}
|
|
catch
|
|
{
|
|
_logger.Warning("CUDA not available, falling back to CPU");
|
|
}
|
|
session = new InferenceSession(modelPath, options);
|
|
_cachedSession = session;
|
|
_cachedModelKey = modelKey;
|
|
// 霈啣�摰鮋�雿輻鍂�?Execution Provider
|
|
var providers = session.ModelMetadata?.CustomMetadataMap;
|
|
_logger.Information("Loaded ONNX model: {ModelPath}, Providers: {Providers}",
|
|
modelPath, string.Join(", ", session.GetType().Name));
|
|
}
|
|
|
|
int h = inputImage.Height;
|
|
int w = inputImage.Width;
|
|
_logger.Information("Input image size: {W}x{H}, Model: {Model}, Scale: {Scale}", w, h, model, scale);
|
|
|
|
// 撖孵之�曆蝙�典��埈綫����伐��踹��閙活�函�餈��/OOM
|
|
const int TileSize = 256;
|
|
bool useTiling = (model.StartsWith("EDSR", StringComparison.OrdinalIgnoreCase)) && (h > TileSize || w > TileSize);
|
|
|
|
if (useTiling)
|
|
{
|
|
return ProcessTiled(session, inputImage, scale, TileSize);
|
|
}
|
|
|
|
return ProcessSingle(session, inputImage, scale);
|
|
}
|
|
|
|
/// <summary>
|
|
/// �閙活�函�嚗���暹� FSRCNN嚗?
|
|
/// </summary>
|
|
private Image<Gray, byte> ProcessSingle(InferenceSession session, Image<Gray, byte> inputImage, int scale)
|
|
{
|
|
int h = inputImage.Height;
|
|
int w = inputImage.Width;
|
|
|
|
// �瑕�璅∪�颲枏�靽⊥�
|
|
string inputName = session.InputMetadata.Keys.First();
|
|
var inputMeta = session.InputMetadata[inputName];
|
|
int[] dims = inputMeta.Dimensions;
|
|
// dims �澆�: [1, H, W, C] (NHWC)嚗龦 �航��?1 �?3
|
|
int inputChannels = dims[^1]; // ���𦒘�蝏湔糓�𡁻��?
|
|
|
|
// ��遣颲枏� tensor: [1, H, W, C] (NHWC)
|
|
// 雿輻鍂摨訫��啁� + Parallel.For �踹��𣂼�蝝删揣撘訫���
|
|
DenseTensor<float> inputTensor;
|
|
if (inputChannels == 1)
|
|
{
|
|
// FSRCNN: �閖�𡁻��啣漲颲枏�
|
|
inputTensor = new DenseTensor<float>(new[] { 1, h, w, 1 });
|
|
float[] buf = inputTensor.Buffer.ToArray();
|
|
var imgData = inputImage.Data;
|
|
Parallel.For(0, h, y =>
|
|
{
|
|
int rowOffset = y * w;
|
|
for (int x = 0; x < w; x++)
|
|
buf[rowOffset + x] = imgData[y, x, 0];
|
|
});
|
|
inputTensor = new DenseTensor<float>(buf, new[] { 1, h, w, 1 });
|
|
}
|
|
else
|
|
{
|
|
// EDSR: 銝厰�𡁻� BGR 颲枏�
|
|
using var colorInput = new Image<Bgr, byte>(w, h);
|
|
CvInvoke.CvtColor(inputImage, colorInput, ColorConversion.Gray2Bgr);
|
|
var buf = new float[h * w * 3];
|
|
var imgData = colorInput.Data;
|
|
Parallel.For(0, h, y =>
|
|
{
|
|
int rowOffset = y * w * 3;
|
|
for (int x = 0; x < w; x++)
|
|
{
|
|
int px = rowOffset + x * 3;
|
|
buf[px] = imgData[y, x, 0];
|
|
buf[px + 1] = imgData[y, x, 1];
|
|
buf[px + 2] = imgData[y, x, 2];
|
|
}
|
|
});
|
|
inputTensor = new DenseTensor<float>(buf, new[] { 1, h, w, 3 });
|
|
}
|
|
|
|
// �函�
|
|
var inputs = new List<NamedOnnxValue>
|
|
{
|
|
NamedOnnxValue.CreateFromTensor(inputName, inputTensor)
|
|
};
|
|
|
|
using var results = session.Run(inputs);
|
|
var outputTensor = results.First().AsTensor<float>();
|
|
|
|
// 颲枏枂 shape: [1, C, H*scale, W*scale] (NCHW嚗峕芋�贝��箇�餈?Transpose)
|
|
var shape = outputTensor.Dimensions;
|
|
int outC = shape[1];
|
|
int outH = shape[2];
|
|
int outW = shape[3];
|
|
|
|
// 頧祆揢銝箇�摨血㦛�?
|
|
// 雿輻鍂 Parallel.For + �湔𦻖����滢�
|
|
Image<Gray, byte> result;
|
|
if (outC == 1)
|
|
{
|
|
// FSRCNN: �閖�𡁻�颲枏枂 [1, 1, outH, outW]
|
|
result = new Image<Gray, byte>(outW, outH);
|
|
var outData = result.Data;
|
|
Parallel.For(0, outH, y =>
|
|
{
|
|
for (int x = 0; x < outW; x++)
|
|
outData[y, x, 0] = (byte)Math.Clamp((int)outputTensor[0, 0, y, x], 0, 255);
|
|
});
|
|
}
|
|
else
|
|
{
|
|
// EDSR: 銝厰�𡁻�颲枏枂 [1, 3, outH, outW] �?�啣漲
|
|
// �湔𦻖霈∠��啣漲�潘�頝唾�銝剝𡢿 BGR �曉����
|
|
result = new Image<Gray, byte>(outW, outH);
|
|
var outData = result.Data;
|
|
Parallel.For(0, outH, y =>
|
|
{
|
|
for (int x = 0; x < outW; x++)
|
|
{
|
|
float b = outputTensor[0, 0, y, x];
|
|
float g = outputTensor[0, 1, y, x];
|
|
float r = outputTensor[0, 2, y, x];
|
|
// BT.601 �啣漲�砍�: 0.299*R + 0.587*G + 0.114*B
|
|
int gray = (int)(0.299f * r + 0.587f * g + 0.114f * b);
|
|
outData[y, x, 0] = (byte)Math.Clamp(gray, 0, 255);
|
|
}
|
|
});
|
|
}
|
|
|
|
_logger.Debug("ProcessSingle: Scale={Scale}, Output={W}x{H}", scale, outW, outH);
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// ����函�嚗�之�?EDSR嚗㚁�撠�㦛�誩��𣂼��堒��急綫����潭𦻖
|
|
/// </summary>
|
|
private Image<Gray, byte> ProcessTiled(InferenceSession session, Image<Gray, byte> inputImage, int scale, int tileSize)
|
|
{
|
|
int h = inputImage.Height;
|
|
int w = inputImage.Width;
|
|
int overlap = 8; // �滚��讐�嚗��撠烐𣄽�亥器蝻䀝憚敶?
|
|
|
|
var result = new Image<Gray, byte>(w * scale, h * scale);
|
|
|
|
int tilesX = (int)Math.Ceiling((double)w / (tileSize - overlap));
|
|
int tilesY = (int)Math.Ceiling((double)h / (tileSize - overlap));
|
|
_logger.Information("Tiled processing: {TilesX}x{TilesY} tiles, tileSize={TileSize}", tilesX, tilesY, tileSize);
|
|
|
|
for (int ty = 0; ty < tilesY; ty++)
|
|
{
|
|
for (int tx = 0; tx < tilesX; tx++)
|
|
{
|
|
int srcX = Math.Min(tx * (tileSize - overlap), w - tileSize);
|
|
int srcY = Math.Min(ty * (tileSize - overlap), h - tileSize);
|
|
srcX = Math.Max(srcX, 0);
|
|
srcY = Math.Max(srcY, 0);
|
|
int tw = Math.Min(tileSize, w - srcX);
|
|
int th = Math.Min(tileSize, h - srcY);
|
|
|
|
// 鋆�� tile
|
|
inputImage.ROI = new System.Drawing.Rectangle(srcX, srcY, tw, th);
|
|
var tile = inputImage.Copy();
|
|
inputImage.ROI = System.Drawing.Rectangle.Empty;
|
|
|
|
// �函��蓥葵 tile
|
|
var srTile = ProcessSingle(session, tile, scale);
|
|
tile.Dispose();
|
|
|
|
// �坔�蝏𤘪�
|
|
int dstX = srcX * scale;
|
|
int dstY = srcY * scale;
|
|
result.ROI = new System.Drawing.Rectangle(dstX, dstY, srTile.Width, srTile.Height);
|
|
srTile.CopyTo(result);
|
|
result.ROI = System.Drawing.Rectangle.Empty;
|
|
srTile.Dispose();
|
|
}
|
|
}
|
|
|
|
_logger.Debug("ProcessTiled: Scale={Scale}, Output={W}x{H}", scale, result.Width, result.Height);
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// �交𪄳璅∪���辣嚗峕�隡睃�蝥扳�蝝Ｗ�銝芰𤌍敶𤏪�.onnx �澆�嚗?
|
|
/// </summary>
|
|
private static string FindModelFile(string model, int scale)
|
|
{
|
|
string baseDir = AppDomain.CurrentDomain.BaseDirectory;
|
|
string fileName = $"{model}_x{scale}.onnx";
|
|
string[] searchPaths = new[]
|
|
{
|
|
Path.Combine(baseDir, "Models", fileName),
|
|
Path.Combine(baseDir, fileName),
|
|
Path.Combine(Directory.GetCurrentDirectory(), "Models", fileName),
|
|
Path.Combine(Directory.GetCurrentDirectory(), fileName),
|
|
};
|
|
|
|
foreach (var path in searchPaths)
|
|
{
|
|
if (File.Exists(path))
|
|
{
|
|
_logger.Debug("Found model file: {Path}", path);
|
|
return path;
|
|
}
|
|
}
|
|
|
|
_logger.Warning("Model file not found: {Model}_x{Scale}.onnx", model, scale);
|
|
return string.Empty;
|
|
}
|
|
} |