#include "Simple3xScaler.hh"
#include "SuperImposedVideoFrame.hh"
#include "LineScalers.hh"
#include "RawFrame.hh"
#include "ScalerOutput.hh"
#include "RenderSettings.hh"
#include "Multiply32.hh"
#include "vla.hh"
#include <cstdint>
#include <memory>
#ifdef __SSE2__
#include <emmintrin.h>
#endif

namespace openmsx {

template <class Pixel> class Blur_1on3
{
public:
	explicit Blur_1on3(const PixelOperations<Pixel>& pixelOps);
	inline void setBlur(unsigned blur_) { blur = blur_; }
	void operator()(const Pixel* in, Pixel* out, size_t dstWidth);
private:
	Multiply32<Pixel> mult0;
	Multiply32<Pixel> mult1;
	Multiply32<Pixel> mult2;
	Multiply32<Pixel> mult3;
	unsigned blur;
#ifdef __SSE2__
	void blur_SSE(const Pixel* in_, Pixel* out_, size_t srcWidth);
#endif
};


template <class Pixel>
Simple3xScaler<Pixel>::Simple3xScaler(
		const PixelOperations<Pixel>& pixelOps_,
		const RenderSettings& settings_)
	: Scaler3<Pixel>(pixelOps_)
	, pixelOps(pixelOps_)
	, scanline(pixelOps_)
	, blur_1on3(std::make_unique<Blur_1on3<Pixel>>(pixelOps_))
	, settings(settings_)
{
}

template <class Pixel>
Simple3xScaler<Pixel>::~Simple3xScaler() = default;

template <typename Pixel>
void Simple3xScaler<Pixel>::doScale1(FrameSource& src,
	unsigned srcStartY, unsigned /*srcEndY*/, unsigned srcWidth,
	ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY,
	PolyLineScaler<Pixel>& scale)
{
	VLA_SSE_ALIGNED(Pixel, buf, srcWidth);
	int scanlineFactor = settings.getScanlineFactor();
	unsigned dstWidth = dst.getWidth();
	unsigned y = dstStartY;
	auto* srcLine = src.getLinePtr(srcStartY++, srcWidth, buf);
	auto* dstLine0 = dst.acquireLine(y + 0);
	scale(srcLine, dstLine0, dstWidth);

	Scale_1on1<Pixel> copy;
	auto* dstLine1 = dst.acquireLine(y + 1);
	copy(dstLine0, dstLine1, dstWidth);

	for (/* */; (y + 4) < dstEndY; y += 3, srcStartY += 1) {
		srcLine = src.getLinePtr(srcStartY, srcWidth, buf);
		auto* dstLine3 = dst.acquireLine(y + 3);
		scale(srcLine, dstLine3, dstWidth);

		auto* dstLine4 = dst.acquireLine(y + 4);
		copy(dstLine3, dstLine4, dstWidth);

		auto* dstLine2 = dst.acquireLine(y + 2);
		scanline.draw(dstLine0, dstLine3, dstLine2,
		              scanlineFactor, dstWidth);

		dst.releaseLine(y + 0, dstLine0);
		dst.releaseLine(y + 1, dstLine1);
		dst.releaseLine(y + 2, dstLine2);
		dstLine0 = dstLine3;
		dstLine1 = dstLine4;
	}
	srcLine = src.getLinePtr(srcStartY, srcWidth, buf);
	VLA_SSE_ALIGNED(Pixel, buf2, dstWidth);
	scale(srcLine, buf2, dstWidth);

	auto* dstLine2 = dst.acquireLine(y + 2);
	scanline.draw(dstLine0, buf2, dstLine2, scanlineFactor, dstWidth);
	dst.releaseLine(y + 0, dstLine0);
	dst.releaseLine(y + 1, dstLine1);
	dst.releaseLine(y + 2, dstLine2);
}

template <typename Pixel>
void Simple3xScaler<Pixel>::doScale2(FrameSource& src,
	unsigned srcStartY, unsigned /*srcEndY*/, unsigned srcWidth,
	ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY,
	PolyLineScaler<Pixel>& scale)
{
	VLA_SSE_ALIGNED(Pixel, buf, srcWidth);
	int scanlineFactor = settings.getScanlineFactor();
	unsigned dstWidth = dst.getWidth();
	for (unsigned srcY = srcStartY, dstY = dstStartY; dstY < dstEndY;
	     srcY += 2, dstY += 3) {
		auto* srcLine0 = src.getLinePtr(srcY + 0, srcWidth, buf);
		auto* dstLine0 = dst.acquireLine(dstY + 0);
		scale(srcLine0, dstLine0, dstWidth);

		auto* srcLine1 = src.getLinePtr(srcY + 1, srcWidth, buf);
		auto* dstLine2 = dst.acquireLine(dstY + 2);
		scale(srcLine1, dstLine2, dstWidth);

		auto* dstLine1 = dst.acquireLine(dstY + 1);
		scanline.draw(dstLine0, dstLine2, dstLine1,
		              scanlineFactor, dstWidth);

		dst.releaseLine(dstY + 0, dstLine0);
		dst.releaseLine(dstY + 1, dstLine1);
		dst.releaseLine(dstY + 2, dstLine2);
	}
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale2x1to9x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_2on9<Pixel>> op(pixelOps);
	doScale1(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale2x2to9x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_2on9<Pixel>> op(pixelOps);
	doScale2(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale1x1to3x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	if (unsigned blur = settings.getBlurFactor() / 3) {
		blur_1on3->setBlur(blur);
		PolyScaleRef<Pixel, Blur_1on3<Pixel>> op(*blur_1on3);
		doScale1(src, srcStartY, srcEndY, srcWidth,
		         dst, dstStartY, dstEndY, op);
	} else {
		// No blurring: this is an optimization but it's also needed
		// for correctness (otherwise there's an overflow in 0.16 fixed
		// point arithmetic).
		PolyScale<Pixel, Scale_1on3<Pixel>> op;
		doScale1(src, srcStartY, srcEndY, srcWidth,
		         dst, dstStartY, dstEndY, op);
	}
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale1x2to3x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_1on3<Pixel>> op;
	doScale2(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale4x1to9x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_4on9<Pixel>> op(pixelOps);
	doScale1(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale4x2to9x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_4on9<Pixel>> op(pixelOps);
	doScale2(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale2x1to3x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_2on3<Pixel>> op(pixelOps);
	doScale1(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale2x2to3x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_2on3<Pixel>> op(pixelOps);
	doScale2(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale8x1to9x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_8on9<Pixel>> op(pixelOps);
	doScale1(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale8x2to9x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_8on9<Pixel>> op(pixelOps);
	doScale2(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale4x1to3x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_4on3<Pixel>> op(pixelOps);
	doScale1(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scale4x2to3x3(FrameSource& src,
		unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	PolyScale<Pixel, Scale_4on3<Pixel>> op(pixelOps);
	doScale2(src, srcStartY, srcEndY, srcWidth, dst, dstStartY, dstEndY, op);
}

template <class Pixel>
void Simple3xScaler<Pixel>::scaleBlank1to3(
		FrameSource& src, unsigned srcStartY, unsigned srcEndY,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	int scanlineFactor = settings.getScanlineFactor();

	unsigned dstHeight = dst.getHeight();
	unsigned stopDstY = (dstEndY == dstHeight)
	                  ? dstEndY : dstEndY - 3;
	unsigned srcY = srcStartY, dstY = dstStartY;
	for (/* */; dstY < stopDstY; srcY += 1, dstY += 3) {
		auto color0 = src.getLineColor<Pixel>(srcY);
		Pixel color1 = scanline.darken(color0, scanlineFactor);
		dst.fillLine(dstY + 0, color0);
		dst.fillLine(dstY + 1, color0);
		dst.fillLine(dstY + 2, color1);
	}
	if (dstY != dstHeight) {
		unsigned nextLineWidth = src.getLineWidth(srcY + 1);
		assert(src.getLineWidth(srcY) == 1);
		assert(nextLineWidth != 1);
		this->dispatchScale(src, srcY, srcEndY, nextLineWidth,
		                    dst, dstY, dstEndY);
	}
}

template <class Pixel>
void Simple3xScaler<Pixel>::scaleBlank2to3(
		FrameSource& src, unsigned srcStartY, unsigned /*srcEndY*/,
		ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	int scanlineFactor = settings.getScanlineFactor();
	for (unsigned srcY = srcStartY, dstY = dstStartY;
	     dstY < dstEndY; srcY += 2, dstY += 3) {
		auto color0 = src.getLineColor<Pixel>(srcY + 0);
		auto color1 = src.getLineColor<Pixel>(srcY + 1);
		Pixel color01 = scanline.darken(color0, color1, scanlineFactor);
		dst.fillLine(dstY + 0, color0);
		dst.fillLine(dstY + 1, color01);
		dst.fillLine(dstY + 2, color1);
	}
}


// class Blur_1on3

template <class Pixel>
Blur_1on3<Pixel>::Blur_1on3(const PixelOperations<Pixel>& pixelOps)
	: mult0(pixelOps)
	, mult1(pixelOps)
	, mult2(pixelOps)
	, mult3(pixelOps)
{
}

#ifdef __SSE2__
template<class Pixel>
void Blur_1on3<Pixel>::blur_SSE(const Pixel* in_, Pixel* out_, size_t srcWidth)
{
	if (sizeof(Pixel) != 4) {
		assert(false); return; // only 32-bpp
	}

	assert((srcWidth % 4) == 0);
	assert(srcWidth >= 8);
	assert((size_t(in_ ) % 16) == 0);
	assert((size_t(out_) % 16) == 0);

	unsigned alpha = blur * 256;
	unsigned c0 = alpha / 2;
	unsigned c1 = alpha + c0;
	unsigned c2 = 0x10000 - c1;
	unsigned c3 = 0x10000 - alpha;
	__m128i C0C1 = _mm_set_epi16(c1, c1, c1, c1, c0, c0, c0, c0);
	__m128i C1C0 = _mm_shuffle_epi32(C0C1, 0x4E);
	__m128i C2C3 = _mm_set_epi16(c3, c3, c3, c3, c2, c2, c2, c2);
	__m128i C3C2 = _mm_shuffle_epi32(C2C3, 0x4E);

	size_t tmp = srcWidth - 4;
	auto* in  = reinterpret_cast<const char*>(in_  +     tmp);
	auto* out = reinterpret_cast<      char*>(out_ + 3 * tmp);
	auto x = -ptrdiff_t(tmp * sizeof(Pixel));

	__m128i ZERO = _mm_setzero_si128();

	// Prepare first iteration (duplicate left border pixel)
	__m128i abcd = _mm_load_si128(reinterpret_cast<const __m128i*>(in + x));
	__m128i a_b_ = _mm_unpacklo_epi8(abcd, ZERO);
	__m128i a_a_ = _mm_unpacklo_epi64(a_b_, a_b_);
	__m128i a0a1 = _mm_mulhi_epu16(a_a_, C0C1);
	__m128i d1d0 = _mm_shuffle_epi32(a0a1, 0x4E); // left border

	// At the start of each iteration the follwoing vars are live:
	//   abcd, a_b_, a_a_, a0a1, d1d0
	// Each iteration reads 4 and produces 12 pixels.
	do {
		// p01
		__m128i a2a3  = _mm_mulhi_epu16(a_a_, C2C3);
		__m128i b_b_  = _mm_unpackhi_epi64(a_b_, a_b_);
		__m128i b1b0  = _mm_mulhi_epu16(b_b_, C1C0);
		__m128i xxb0  = _mm_unpackhi_epi64(ZERO, b1b0);
		__m128i p01   = _mm_add_epi16(_mm_add_epi16(d1d0, a2a3), xxb0);
		// p23
		__m128i xxa1  = _mm_unpackhi_epi64(ZERO, a0a1);
		__m128i b3b2  = _mm_mulhi_epu16(b_b_, C3C2);
		__m128i a2b2  = shuffle<0xE4>(a2a3, b3b2);
		__m128i b1xx  = _mm_unpacklo_epi64(b1b0, ZERO);
		__m128i p23   = _mm_add_epi16(_mm_add_epi16(xxa1, a2b2), b1xx);
		__m128i p0123 = _mm_packus_epi16(p01, p23);
		_mm_store_si128(reinterpret_cast<__m128i*>(out + 3 * x + 0),
		                p0123);

		// p45
		__m128i a0xx  = _mm_unpacklo_epi64(a0a1, ZERO);
		__m128i c_d_  = _mm_unpackhi_epi8(abcd, ZERO);
		__m128i c_c_  = _mm_unpacklo_epi64(c_d_, c_d_);
		__m128i c0c1  = _mm_mulhi_epu16(c_c_, C0C1);
		__m128i p45   = _mm_add_epi16(_mm_add_epi16(a0xx, b3b2), c0c1);
		// p67
		__m128i c2c3  = _mm_mulhi_epu16(c_c_, C2C3);
		__m128i d_d_  = _mm_unpackhi_epi64(c_d_, c_d_);
		        d1d0  = _mm_mulhi_epu16(d_d_, C1C0);
		__m128i xxd0  = _mm_unpackhi_epi64(ZERO, d1d0);
		__m128i p67   = _mm_add_epi16(_mm_add_epi16(b1b0, c2c3), xxd0);
		__m128i p4567 = _mm_packus_epi16(p45, p67);
		_mm_store_si128(reinterpret_cast<__m128i*>(out + 3 * x + 16),
		                p4567);

		// p89
		__m128i xxc1  = _mm_unpackhi_epi64(ZERO, c0c1);
		__m128i d3d2  = _mm_mulhi_epu16(d_d_, C3C2);
		__m128i c2d2  = shuffle<0xE4>(c2c3, d3d2);
		__m128i d1xx  = _mm_unpacklo_epi64(d1d0, ZERO);
		__m128i p89   = _mm_add_epi16(_mm_add_epi16(xxc1, c2d2), d1xx);
		// pab
		__m128i c0xx  = _mm_unpacklo_epi64(c0c1, ZERO);
		        abcd  = _mm_load_si128(reinterpret_cast<const __m128i*>(in + x + 16));
		        a_b_  = _mm_unpacklo_epi8(abcd, ZERO);
		        a_a_  = _mm_unpacklo_epi64(a_b_, a_b_);
		        a0a1  = _mm_mulhi_epu16(a_a_, C0C1);
		__m128i pab   = _mm_add_epi16(_mm_add_epi16(c0xx, d3d2), a0a1);
		__m128i p89ab = _mm_packus_epi16(p89, pab);
		_mm_store_si128(reinterpret_cast<__m128i*>(out + 3 * x + 32),
		                p89ab);

		x += 16;
	} while (x < 0);

	// Last iteration (duplicate right border pixel)
	// p01
	__m128i a2a3  = _mm_mulhi_epu16(a_a_, C2C3);
	__m128i b_b_  = _mm_unpackhi_epi64(a_b_, a_b_);
	__m128i b1b0  = _mm_mulhi_epu16(b_b_, C1C0);
	__m128i xxb0  = _mm_unpackhi_epi64(ZERO, b1b0);
	__m128i p01   = _mm_add_epi16(_mm_add_epi16(d1d0, a2a3), xxb0);
	// p23
	__m128i xxa1  = _mm_unpackhi_epi64(ZERO, a0a1);
	__m128i b3b2  = _mm_mulhi_epu16(b_b_, C3C2);
	__m128i a2b2  = shuffle<0xE4>(a2a3, b3b2);
	__m128i b1xx  = _mm_unpacklo_epi64(b1b0, ZERO);
	__m128i p23   = _mm_add_epi16(_mm_add_epi16(xxa1, a2b2), b1xx);
	__m128i p0123 = _mm_packus_epi16(p01, p23);
	_mm_store_si128(reinterpret_cast<__m128i*>(out + 0),
	                p0123);

	// p45
	__m128i a0xx  = _mm_unpacklo_epi64(a0a1, ZERO);
	__m128i c_d_  = _mm_unpackhi_epi8(abcd, ZERO);
	__m128i c_c_  = _mm_unpacklo_epi64(c_d_, c_d_);
	__m128i c0c1  = _mm_mulhi_epu16(c_c_, C0C1);
	__m128i p45   = _mm_add_epi16(_mm_add_epi16(a0xx, b3b2), c0c1);
	// p67
	__m128i c2c3  = _mm_mulhi_epu16(c_c_, C2C3);
	__m128i d_d_  = _mm_unpackhi_epi64(c_d_, c_d_);
		d1d0  = _mm_mulhi_epu16(d_d_, C1C0);
	__m128i xxd0  = _mm_unpackhi_epi64(ZERO, d1d0);
	__m128i p67   = _mm_add_epi16(_mm_add_epi16(b1b0, c2c3), xxd0);
	__m128i p4567 = _mm_packus_epi16(p45, p67);
	_mm_store_si128(reinterpret_cast<__m128i*>(out + 16),
	                p4567);

	// p89
	__m128i xxc1  = _mm_unpackhi_epi64(ZERO, c0c1);
	__m128i d3d2  = _mm_mulhi_epu16(d_d_, C3C2);
	__m128i c2d2  = shuffle<0xE4>(c2c3, d3d2);
	__m128i d1xx  = _mm_unpacklo_epi64(d1d0, ZERO);
	__m128i p89   = _mm_add_epi16(_mm_add_epi16(xxc1, c2d2), d1xx);
	// pab
	__m128i c0xx  = _mm_unpacklo_epi64(c0c1, ZERO);
	        a0a1  = _mm_shuffle_epi32(d1d0, 0x4E); // right border
	__m128i pab   = _mm_add_epi16(_mm_add_epi16(c0xx, d3d2), a0a1);
	__m128i p89ab = _mm_packus_epi16(p89, pab);
	_mm_store_si128(reinterpret_cast<__m128i*>(out + 32),
	                p89ab);
}
#endif

template <class Pixel>
void Blur_1on3<Pixel>::operator()(
	const Pixel* __restrict in, Pixel* __restrict out,
	size_t dstWidth)
{
	/* The following code is equivalent to this loop. It is 2x unrolled
	 * and common subexpressions have been eliminated. The last iteration
	 * is also moved outside the for loop.
	 *
	 *  unsigned c0 = blur / 2;
	 *  unsigned c1 = c0 + blur;
	 *  unsigned c2 = 256 - c1;
	 *  unsigned c3 = 256 - 2 * c0;
	 *  Pixel prev, curr, next;
	 *  prev = curr = next = in[0];
	 *  size_t srcWidth = dstWidth / 3;
	 *  for (unsigned x = 0; x < srcWidth; ++x) {
	 *      if (x != (srcWidth - 1)) next = in[x + 1];
	 *      out[3 * x + 0] = mul(c1, prev) + mul(c2, curr);
	 *      out[3 * x + 1] = mul(c0, prev) + mul(c3, curr) + mul(c0, next);
	 *      out[3 * x + 2] =                 mul(c2, curr) + mul(c1, next);
	 *      prev = curr;
	 *      curr = next;
	 *  }
	 */
	size_t srcWidth = dstWidth / 3;
#ifdef __SSE2__
	if (sizeof(Pixel) == 4) {
		blur_SSE(in, out, srcWidth);
		return;
	}
#endif

	// C++ routine, both 16bpp and 32bpp
	unsigned c0 = blur / 2;
	unsigned c1 = blur + c0;
	unsigned c2 = 256 - c1;
	unsigned c3 = 256 - 2 * c0;
	mult0.setFactor32(c0);
	mult1.setFactor32(c1);
	mult2.setFactor32(c2);
	mult3.setFactor32(c3);

	Pixel p0 = in[0];
	Pixel p1;
	uint32_t f0 = mult0.mul32(p0);
	uint32_t f1 = mult1.mul32(p0);
	uint32_t g0 = f0;
	uint32_t g1 = f1;

	size_t x;
	for (x = 0; x < (srcWidth - 2); x += 2) {
		uint32_t g2 = mult2.mul32(p0);
		out[3 * x + 0] = mult0.conv32(g2 + f1);
		p1 = in[x + 1];
		uint32_t t0 = mult0.mul32(p1);
		out[3 * x + 1] = mult0.conv32(f0 + mult3.mul32(p0) + t0);
		f0 = t0;
		f1 = mult1.mul32(p1);
		out[3 * x + 2] = mult0.conv32(g2 + f1);

		uint32_t f2 = mult2.mul32(p1);
		out[3 * x + 3] = mult0.conv32(f2 + g1);
		p0 = in[x + 2];
		uint32_t t1 = mult0.mul32(p0);
		out[3 * x + 4] = mult0.conv32(g0 + mult3.mul32(p1) + t1);
		g0 = t1;
		g1 = mult1.mul32(p0);
		out[3 * x + 5] = mult0.conv32(g1 + f2);
	}
	uint32_t g2 = mult2.mul32(p0);
	out[3 * x + 0] = mult0.conv32(g2 + f1);
	p1 = in[x + 1];
	uint32_t t0 = mult0.mul32(p1);
	out[3 * x + 1] = mult0.conv32(f0 + mult3.mul32(p0) + t0);
	f0 = t0;
	f1 = mult1.mul32(p1);
	out[3 * x + 2] = mult0.conv32(g2 + f1);

	uint32_t f2 = mult2.mul32(p1);
	out[3 * x + 3] = mult0.conv32(f2 + g1);
	out[3 * x + 4] = mult0.conv32(g0 + mult3.mul32(p1) + f0);
	out[3 * x + 5] = p1;
}

template <class Pixel>
void Simple3xScaler<Pixel>::scaleImage(
	FrameSource& src, const RawFrame* superImpose,
	unsigned srcStartY, unsigned srcEndY, unsigned srcWidth,
	ScalerOutput<Pixel>& dst, unsigned dstStartY, unsigned dstEndY)
{
	if (superImpose) {
		SuperImposedVideoFrame<Pixel> sf(src, *superImpose, pixelOps);
		srcWidth = sf.getLineWidth(srcStartY);
		this->dispatchScale(sf,  srcStartY, srcEndY, srcWidth,
		                    dst, dstStartY, dstEndY);
	} else {
		this->dispatchScale(src, srcStartY, srcEndY, srcWidth,
		                    dst, dstStartY, dstEndY);
	}
}

// Force template instantiation.
#if HAVE_16BPP
template class Simple3xScaler<uint16_t>;
#endif
#if HAVE_32BPP
template class Simple3xScaler<uint32_t>;
#endif

} // namespace openmsx
