Separate out CREG packing into new pattern, to avoid conflict with PREG

4 files changed, 273 insertions, 46 deletions

diff --git a/passes/pmgen/Makefile.inc b/passes/pmgen/Makefile.inc
index 82bb40ac8..366c37943 100644
--- a/passes/pmgen/Makefile.inc
+++ b/passes/pmgen/Makefile.inc
@@ -22,8 +22,9 @@ $(eval $(call add_extra_objs,passes/pmgen/ice40_wrapcarry_pm.h))
 # --------------------------------------
 
 OBJS += passes/pmgen/xilinx_dsp.o
-passes/pmgen/xilinx_dsp.o: passes/pmgen/xilinx_dsp_pm.h passes/pmgen/xilinx_dsp_cascade_pm.h
+passes/pmgen/xilinx_dsp.o: passes/pmgen/xilinx_dsp_pm.h passes/pmgen/xilinx_dsp_CREG_pm.h passes/pmgen/xilinx_dsp_cascade_pm.h
 $(eval $(call add_extra_objs,passes/pmgen/xilinx_dsp_pm.h))
+$(eval $(call add_extra_objs,passes/pmgen/xilinx_dsp_CREG_pm.h))
 $(eval $(call add_extra_objs,passes/pmgen/xilinx_dsp_cascade_pm.h))
 
 # --------------------------------------
diff --git a/passes/pmgen/xilinx_dsp.cc b/passes/pmgen/xilinx_dsp.cc
index 9d0a77e2b..86472feb5 100644
--- a/passes/pmgen/xilinx_dsp.cc
+++ b/passes/pmgen/xilinx_dsp.cc
@@ -27,6 +27,7 @@ PRIVATE_NAMESPACE_BEGIN
 bool did_something;
 
 #include "passes/pmgen/xilinx_dsp_pm.h"
+#include "passes/pmgen/xilinx_dsp_CREG_pm.h"
 #include "passes/pmgen/xilinx_dsp_cascade_pm.h"
 
 static Cell* addDsp(Module *module) {
@@ -63,7 +64,7 @@ static Cell* addDsp(Module *module) {
 	return cell;
 }
 
-void pack_xilinx_simd(Module *module, const std::vector<Cell*> &selected_cells)
+void xilinx_simd_pack(Module *module, const std::vector<Cell*> &selected_cells)
 {
 	std::deque<Cell*> simd12_add, simd12_sub;
 	std::deque<Cell*> simd24_add, simd24_sub;
@@ -255,21 +256,18 @@ void pack_xilinx_simd(Module *module, const std::vector<Cell*> &selected_cells)
 	g24(simd24_sub);
 }
 
-
-void pack_xilinx_dsp(xilinx_dsp_pm &pm)
+void xilinx_dsp_pack(xilinx_dsp_pm &pm)
 {
 	auto &st = pm.st_xilinx_dsp_pack;
 
 	log("Analysing %s.%s for Xilinx DSP packing.\n", log_id(pm.module), log_id(st.dsp));
 
-	log_debug("\n");
 	log_debug("preAdd:     %s\n", log_id(st.preAdd, "--"));
 	log_debug("ffAD:       %s %s %s\n", log_id(st.ffAD, "--"), log_id(st.ffADcemux, "--"), log_id(st.ffADrstmux, "--"));
 	log_debug("ffA2:       %s %s %s\n", log_id(st.ffA2, "--"), log_id(st.ffA2cemux, "--"), log_id(st.ffA2rstmux, "--"));
 	log_debug("ffA1:       %s %s %s\n", log_id(st.ffA1, "--"), log_id(st.ffA1cemux, "--"), log_id(st.ffA1rstmux, "--"));
 	log_debug("ffB2:       %s %s %s\n", log_id(st.ffB2, "--"), log_id(st.ffB2cemux, "--"), log_id(st.ffB2rstmux, "--"));
 	log_debug("ffB1:       %s %s %s\n", log_id(st.ffB1, "--"), log_id(st.ffB1cemux, "--"), log_id(st.ffB1rstmux, "--"));
-	log_debug("ffC:        %s %s %s\n", log_id(st.ffC, "--"), log_id(st.ffCcemux, "--"), log_id(st.ffCrstmux, "--"));
 	log_debug("ffD:        %s %s %s\n", log_id(st.ffD, "--"), log_id(st.ffDcemux, "--"), log_id(st.ffDrstmux, "--"));
 	log_debug("dsp:        %s\n", log_id(st.dsp, "--"));
 	log_debug("ffM:        %s %s %s\n", log_id(st.ffM, "--"), log_id(st.ffMcemux, "--"), log_id(st.ffMrstmux, "--"));
@@ -277,6 +275,7 @@ void pack_xilinx_dsp(xilinx_dsp_pm &pm)
 	log_debug("postAddMux: %s\n", log_id(st.postAddMux, "--"));
 	log_debug("ffP:        %s %s %s\n", log_id(st.ffP, "--"), log_id(st.ffPcemux, "--"), log_id(st.ffPrstmux, "--"));
 	log_debug("overflow:   %s\n", log_id(st.overflow, "--"));
+	log_debug("\n");
 
 	Cell *cell = st.dsp;
 
@@ -426,12 +425,6 @@ void pack_xilinx_dsp(xilinx_dsp_pm &pm)
 			else
 				cell->setParam(ID(BREG), 1);
 		}
-		if (st.ffC) {
-			SigSpec &C = cell->connections_.at(ID(C));
-			f(C, st.ffC, st.ffCcemux, st.ffCcepol, ID(CEC), st.ffCrstmux, st.ffCrstpol, ID(RSTC));
-			pm.add_siguser(C, cell);
-			cell->setParam(ID(CREG), 1);
-		}
 		if (st.ffD) {
 			SigSpec &D = cell->connections_.at(ID(D));
 			f(D, st.ffD, st.ffDcemux, st.ffDcepol, ID(CED), st.ffDrstmux, st.ffDrstpol, ID(RSTD));
@@ -468,9 +461,6 @@ void pack_xilinx_dsp(xilinx_dsp_pm &pm)
 				log(" ffB1:%s", log_id(st.ffB1));
 		}
 
-		if (st.ffC)
-			log(" ffC:%s", log_id(st.ffC));
-
 		if (st.ffD)
 			log(" ffD:%s", log_id(st.ffD));
 
@@ -491,6 +481,76 @@ void pack_xilinx_dsp(xilinx_dsp_pm &pm)
 	pm.blacklist(cell);
 }
 
+void xilinx_dsp_packC(xilinx_dsp_CREG_pm &pm)
+{
+	auto &st = pm.st_xilinx_dsp_packC;
+
+	log_debug("Analysing %s.%s for Xilinx DSP packing (CREG).\n", log_id(pm.module), log_id(st.dsp));
+	log_debug("ffC:        %s %s %s\n", log_id(st.ffC, "--"), log_id(st.ffCcemux, "--"), log_id(st.ffCrstmux, "--"));
+	log_debug("\n");
+
+	Cell *cell = st.dsp;
+
+	if (st.clock != SigBit())
+	{
+		cell->setPort(ID(CLK), st.clock);
+
+		auto f = [&pm,cell](SigSpec &A, Cell* ff, Cell* cemux, bool cepol, IdString ceport, Cell* rstmux, bool rstpol, IdString rstport) {
+			SigSpec D = ff->getPort(ID(D));
+			SigSpec Q = pm.sigmap(ff->getPort(ID(Q)));
+			if (!A.empty())
+				A.replace(Q, D);
+			if (rstmux) {
+				SigSpec Y = rstmux->getPort(ID(Y));
+				SigSpec AB = rstmux->getPort(rstpol ? ID(A) : ID(B));
+				if (!A.empty())
+					A.replace(Y, AB);
+				if (rstport != IdString()) {
+					SigSpec S = rstmux->getPort(ID(S));
+					cell->setPort(rstport, rstpol ? S : pm.module->Not(NEW_ID, S));
+				}
+			}
+			else if (rstport != IdString())
+				cell->setPort(rstport, State::S0);
+			if (cemux) {
+				SigSpec Y = cemux->getPort(ID(Y));
+				SigSpec BA = cemux->getPort(cepol ? ID(B) : ID(A));
+				SigSpec S = cemux->getPort(ID(S));
+				if (!A.empty())
+					A.replace(Y, BA);
+				cell->setPort(ceport, cepol ? S : pm.module->Not(NEW_ID, S));
+			}
+			else
+				cell->setPort(ceport, State::S1);
+
+			for (auto c : Q.chunks()) {
+				auto it = c.wire->attributes.find(ID(init));
+				if (it == c.wire->attributes.end())
+					continue;
+				for (int i = c.offset; i < c.offset+c.width; i++) {
+					log_assert(it->second[i] == State::S0 || it->second[i] == State::Sx);
+					it->second[i] = State::Sx;
+				}
+			}
+		};
+
+		if (st.ffC) {
+			SigSpec &C = cell->connections_.at(ID(C));
+			f(C, st.ffC, st.ffCcemux, st.ffCcepol, ID(CEC), st.ffCrstmux, st.ffCrstpol, ID(RSTC));
+			pm.add_siguser(C, cell);
+			cell->setParam(ID(CREG), 1);
+		}
+
+		log("  clock: %s (%s)", log_signal(st.clock), "posedge");
+
+		if (st.ffC)
+			log(" ffC:%s", log_id(st.ffC));
+		log("\n");
+	}
+
+	pm.blacklist(cell);
+}
+
 struct XilinxDspPass : public Pass {
 	XilinxDspPass() : Pass("xilinx_dsp", "Xilinx: pack resources into DSPs") { }
 	void help() YS_OVERRIDE
@@ -540,17 +600,23 @@ struct XilinxDspPass : public Pass {
 		extra_args(args, argidx, design);
 
 		for (auto module : design->selected_modules()) {
-			pack_xilinx_simd(module, module->selected_cells());
+			xilinx_simd_pack(module, module->selected_cells());
 
-			xilinx_dsp_pm pm(module, module->selected_cells());
-			pm.run_xilinx_dsp_pack(pack_xilinx_dsp);
+            {
+                xilinx_dsp_pm pm(module, module->selected_cells());
+                pm.run_xilinx_dsp_pack(xilinx_dsp_pack);
+            }
+            {
+                xilinx_dsp_CREG_pm pm(module, module->selected_cells());
+                pm.run_xilinx_dsp_packC(xilinx_dsp_packC);
+            }
 
 			do {
 				did_something = false;
-				xilinx_dsp_cascade_pm pmc(module, module->selected_cells());
-				pmc.run_xilinx_dsp_cascadeP();
-				//pmc.run_xilinx_dsp_cascadeAB();
-				break;
+				xilinx_dsp_cascade_pm pm(module, module->selected_cells());
+				pm.run_xilinx_dsp_cascadeP();
+				//pm.run_xilinx_dsp_cascadeAB();
+                break;
 			} while (did_something);
 		}
 	}
diff --git a/passes/pmgen/xilinx_dsp.pmg b/passes/pmgen/xilinx_dsp.pmg
index 80bf775bc..553195649 100644
--- a/passes/pmgen/xilinx_dsp.pmg
+++ b/passes/pmgen/xilinx_dsp.pmg
@@ -4,11 +4,11 @@ udata <std::function<SigSpec(const SigSpec&)>> unextend
 state <SigBit> clock
 state <SigSpec> sigA sigB sigC sigD sigM sigP
 state <IdString> postAddAB postAddMuxAB
-state <bool> ffA1cepol ffA2cepol ffADcepol ffB1cepol ffB2cepol ffCcepol ffDcepol ffMcepol ffPcepol
-state <bool> ffArstpol ffADrstpol ffBrstpol ffCrstpol ffDrstpol ffMrstpol ffPrstpol
+state <bool> ffA1cepol ffA2cepol ffADcepol ffB1cepol ffB2cepol ffDcepol ffMcepol ffPcepol
+state <bool> ffArstpol ffADrstpol ffBrstpol ffDrstpol ffMrstpol ffPrstpol
 
 state <Cell*> ffAD ffADcemux ffADrstmux ffA1 ffA1cemux ffA1rstmux ffA2 ffA2cemux ffA2rstmux
-state <Cell*> ffB1 ffB1cemux ffB1rstmux ffB2 ffB2cemux ffB2rstmux ffC ffCcemux ffCrstmux
+state <Cell*> ffB1 ffB1cemux ffB1rstmux ffB2 ffB2cemux ffB2rstmux
 state <Cell*> ffD ffDcemux ffDrstmux ffM ffMcemux ffMrstmux ffP ffPcemux ffPrstmux
 
 // subpattern
@@ -24,7 +24,7 @@ match dsp
 	select dsp->type.in(\DSP48E1)
 endmatch
 
-code sigA sigB sigC sigD sigM
+code sigA sigB sigC sigD sigM clock
 	unextend = [](const SigSpec &sig) {
 		int i;
 		for (i = GetSize(sig)-1; i > 0; i--)
@@ -54,6 +54,8 @@ code sigA sigB sigC sigD sigM
 	}
 	else
 		sigM = P;
+
+	clock = port(dsp, \CLK, SigBit());
 endcode
 
 code argQ ffAD ffADcemux ffADrstmux ffADcepol ffADrstpol sigA clock
@@ -326,26 +328,6 @@ code sigC
 		sigC = port(postAddMux, postAddMuxAB == \A ? \B : \A);
 endcode
 
-code argQ ffC ffCcemux ffCrstmux ffCcepol ffCrstpol sigC clock
-	if (param(dsp, \CREG).as_int() == 0 && sigC != sigP) {
-		argQ = sigC;
-		subpattern(in_dffe);
-		if (dff) {
-			ffC = dff;
-			clock = dffclock;
-			if (dffrstmux) {
-				ffCrstmux = dffrstmux;
-				ffCrstpol = dffrstpol;
-			}
-			if (dffcemux) {
-				ffCcemux = dffcemux;
-				ffCcepol = dffcepol;
-			}
-			sigC = dffD;
-		}
-	}
-endcode
-
 match overflow
 	if ffP
 	if param(dsp, \USE_PATTERN_DETECT, Const("NO_PATDET")).decode_string() == "NO_PATDET"
diff --git a/passes/pmgen/xilinx_dsp_CREG.pmg b/passes/pmgen/xilinx_dsp_CREG.pmg
new file mode 100644
index 000000000..d79abdd4a
--- /dev/null
+++ b/passes/pmgen/xilinx_dsp_CREG.pmg
@@ -0,0 +1,178 @@
+pattern xilinx_dsp_packC
+
+udata <std::function<SigSpec(const SigSpec&)>> unextend
+state <SigBit> clock
+state <SigSpec> sigC sigP
+state <bool> ffCcepol ffCrstpol
+state <Cell*> ffC ffCcemux ffCrstmux
+
+// subpattern
+state <SigSpec> argQ argD
+state <bool> ffcepol ffrstpol
+state <int> ffoffset
+udata <SigSpec> dffD dffQ
+udata <SigBit> dffclock
+udata <Cell*> dff dffcemux dffrstmux
+udata <bool> dffcepol dffrstpol
+
+match dsp
+	select dsp->type.in(\DSP48E1)
+	select param(dsp, \CREG, 1).as_int() == 0
+	select nusers(port(dsp, \C, SigSpec())) > 1
+endmatch
+
+code argQ ffC ffCcemux ffCrstmux ffCcepol ffCrstpol sigC sigP clock
+	unextend = [](const SigSpec &sig) {
+		int i;
+		for (i = GetSize(sig)-1; i > 0; i--)
+			if (sig[i] != sig[i-1])
+				break;
+		// Do not remove non-const sign bit
+		if (sig[i].wire)
+			++i;
+		return sig.extract(0, i);
+	};
+	sigC = unextend(port(dsp, \C, SigSpec()));
+
+	SigSpec P = port(dsp, \P);
+	if (param(dsp, \USE_MULT, Const("MULTIPLY")).decode_string() == "MULTIPLY") {
+		// Only care about those bits that are used
+		int i;
+		for (i = 0; i < GetSize(P); i++) {
+			if (nusers(P[i]) <= 1)
+				break;
+			sigP.append(P[i]);
+		}
+		log_assert(nusers(P.extract_end(i)) <= 1);
+	}
+	else
+		sigP = P;
+
+	if (sigC == sigP)
+		reject;
+
+	clock = port(dsp, \CLK, SigBit());
+
+	argQ = sigC;
+	subpattern(in_dffe);
+	if (dff) {
+		ffC = dff;
+		clock = dffclock;
+		if (dffrstmux) {
+			ffCrstmux = dffrstmux;
+			ffCrstpol = dffrstpol;
+		}
+		if (dffcemux) {
+			ffCcemux = dffcemux;
+			ffCcepol = dffcepol;
+		}
+		sigC = dffD;
+	}
+endcode
+
+code
+	if (ffC)
+		accept;
+endcode
+
+// #######################
+
+subpattern in_dffe
+arg argD argQ clock
+
+code
+	dff = nullptr;
+	for (auto c : argQ.chunks()) {
+		if (!c.wire)
+			reject;
+		if (c.wire->get_bool_attribute(\keep))
+			reject;
+	}
+endcode
+
+match ff
+	select ff->type.in($dff)
+	// DSP48E1 does not support clock inversion
+	select param(ff, \CLK_POLARITY).as_bool()
+
+	slice offset GetSize(port(ff, \D))
+	index <SigBit> port(ff, \Q)[offset] === argQ[0]
+
+	// Check that the rest of argQ is present
+	filter GetSize(port(ff, \Q)) >= offset + GetSize(argQ)
+	filter port(ff, \Q).extract(offset, GetSize(argQ)) == argQ
+
+	set ffoffset offset
+endmatch
+
+code argQ argD
+{
+	if (clock != SigBit() && port(ff, \CLK) != clock)
+		reject;
+
+	SigSpec Q = port(ff, \Q);
+	dff = ff;
+	dffclock = port(ff, \CLK);
+	dffD = argQ;
+	argD = port(ff, \D);
+	argQ = Q;
+	dffD.replace(argQ, argD);
+	// Only search for ffrstmux if dffD only
+	//   has two (ff, ffrstmux) users
+	if (nusers(dffD) > 2)
+		argD = SigSpec();
+}
+endcode
+
+match ffrstmux
+	if !argD.empty()
+	select ffrstmux->type.in($mux)
+	index <SigSpec> port(ffrstmux, \Y) === argD
+
+	choice <IdString> BA {\B, \A}
+	// DSP48E1 only supports reset to zero
+	select port(ffrstmux, BA).is_fully_zero()
+
+	define <bool> pol (BA == \B)
+	set ffrstpol pol
+	semioptional
+endmatch
+
+code argD
+	if (ffrstmux) {
+		dffrstmux = ffrstmux;
+		dffrstpol = ffrstpol;
+		argD = port(ffrstmux, ffrstpol ? \A : \B);
+		dffD.replace(port(ffrstmux, \Y), argD);
+
+		// Only search for ffcemux if argQ has at
+		//   least 3 users (ff, <upstream>, ffrstmux) and
+		//   dffD only has two (ff, ffrstmux)
+		if (!(nusers(argQ) >= 3 && nusers(dffD) == 2))
+			argD = SigSpec();
+	}
+	else
+		dffrstmux = nullptr;
+endcode
+
+match ffcemux
+	if !argD.empty()
+	select ffcemux->type.in($mux)
+	index <SigSpec> port(ffcemux, \Y) === argD
+	choice <IdString> AB {\A, \B}
+	index <SigSpec> port(ffcemux, AB) === argQ
+	define <bool> pol (AB == \A)
+	set ffcepol pol
+	semioptional
+endmatch
+
+code argD
+	if (ffcemux) {
+		dffcemux = ffcemux;
+		dffcepol = ffcepol;
+		argD = port(ffcemux, ffcepol ? \B : \A);
+		dffD.replace(port(ffcemux, \Y), argD);
+	}
+	else
+		dffcemux = nullptr;
+endcode